Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+eval_nbest_predictions.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,274 @@
+---
+language: fr
+license: mit
+tags:
+- deberta-v2
+- question-answering
+base_model: almanach/camembertav2-base
+datasets:
+- FQuAD
+metrics:
+- accuracy
+pipeline_tag: text-classification
+library_name: transformers
+model-index:
+- name: almanach/camembertav2-base-fquad
+  results:
+  - task:
+      type: text-classification
+      name: Natural Language Inference
+    dataset:
+      type: FQuAD
+      name: FQuAD
+    metrics:
+    - name: accuracy
+      type: accuracy
+      value: 
+      verified: false
+---
+
+# Model Card for almanach/camembertav2-base-fquad
+
+almanach/camembertav2-base-fquad is a deberta-v2 model for question answering. It is trained on the FQuAD dataset for the task of Extractive Question Answering. The model achieves an f1-score of 83.36016 on the FQuAD dataset.
+
+The model is part of the almanach/camembertav2-base family of model finetunes.
+
+## Model Details
+
+### Model Description
+
+- **Developed by:** Wissam Antoun (Phd Student at Almanach, Inria-Paris)
+- **Model type:** deberta-v2
+- **Language(s) (NLP):** French
+- **License:** MIT
+- **Finetuned from model [optional]:** almanach/camembertav2-base
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** https://github.com/WissamAntoun/camemberta
+- **Paper:** https://arxiv.org/abs/2411.08868
+
+## Uses
+
+The model can be used for question answering tasks in French for Extractive Question Answering.
+
+## Bias, Risks, and Limitations
+
+The model may exhibit biases based on the training data. The model may not generalize well to other datasets or tasks. The model may also have limitations in terms of the data it was trained on.
+
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+```python
+from transformers import AutoTokenizer, AutoModelForQuestionAnswering, pipeline
+
+model = AutoModelForQuestionAnswering.from_pretrained("almanach/camembertav2-base-fquad")
+tokenizer = AutoTokenizer.from_pretrained("almanach/camembertav2-base-fquad")
+
+classifier = pipeline("question-answering", model=model, tokenizer=tokenizer)
+
+classifier(question="Quelle est la capitale de la France ?", context="La capitale de la France est Paris.")
+```
+
+
+## Training Details
+
+### Training Data
+
+The model is trained on the FQuAD dataset.
+
+- Dataset Name: FQuAD
+- Dataset Size:
+    - Train: 20731
+    - Dev: 3188
+
+
+### Training Procedure
+
+Model trained with the run_qa.py script from the huggingface repository.
+
+
+
+#### Training Hyperparameters
+
+```yml
+'Unnamed: 0': /scratch/camembertv2/runs/results/fquad/camembertav2-base-bf16-p2-17000/max_seq_length-896-doc_stride-128-max_answer_length-30-gradient_accumulation_steps-2-precision-fp32-learning_rate-3e-05-epochs-6-lr_scheduler-cosine-warmup_steps-0/SEED-1/all_results.json
+accelerator_config: '{''split_batches'': False, ''dispatch_batches'': None, ''even_batches'':
+  True, ''use_seedable_sampler'': True, ''non_blocking'': False, ''gradient_accumulation_kwargs'':
+  None}'
+adafactor: false
+adam_beta1: 0.9
+adam_beta2: 0.999
+adam_epsilon: 1.0e-08
+auto_find_batch_size: false
+base_model: camembertv2
+base_model_name: camembertav2-base-bf16-p2-17000
+batch_eval_metrics: false
+bf16: false
+bf16_full_eval: false
+data_seed: 1.0
+dataloader_drop_last: false
+dataloader_num_workers: 0
+dataloader_persistent_workers: false
+dataloader_pin_memory: true
+dataloader_prefetch_factor: .nan
+ddp_backend: .nan
+ddp_broadcast_buffers: .nan
+ddp_bucket_cap_mb: .nan
+ddp_find_unused_parameters: .nan
+ddp_timeout: 1800
+debug: '[]'
+deepspeed: .nan
+disable_tqdm: false
+dispatch_batches: .nan
+do_eval: true
+do_predict: false
+do_train: true
+epoch: 6.0
+eval_accumulation_steps: 1
+eval_delay: 0
+eval_do_concat_batches: true
+eval_exact_match: 64.42910915934755
+eval_f1: 83.36016013340664
+eval_on_start: false
+eval_runtime: 45.7589
+eval_samples: 3188.0
+eval_samples_per_second: 69.669
+eval_steps: .nan
+eval_steps_per_second: 1.093
+eval_strategy: epoch
+eval_use_gather_object: false
+evaluation_strategy: epoch
+fp16: false
+fp16_backend: auto
+fp16_full_eval: false
+fp16_opt_level: O1
+fsdp: '[]'
+fsdp_config: '{''min_num_params'': 0, ''xla'': False, ''xla_fsdp_v2'': False, ''xla_fsdp_grad_ckpt'':
+  False}'
+fsdp_min_num_params: 0
+fsdp_transformer_layer_cls_to_wrap: .nan
+full_determinism: false
+gradient_accumulation_steps: 2
+gradient_checkpointing: false
+gradient_checkpointing_kwargs: .nan
+greater_is_better: true
+group_by_length: false
+half_precision_backend: auto
+hub_always_push: false
+hub_model_id: .nan
+hub_private_repo: false
+hub_strategy: every_save
+hub_token: <HUB_TOKEN>
+ignore_data_skip: false
+include_inputs_for_metrics: false
+include_num_input_tokens_seen: false
+include_tokens_per_second: false
+jit_mode_eval: false
+label_names: .nan
+label_smoothing_factor: 0.0
+learning_rate: 3.0e-05
+length_column_name: length
+load_best_model_at_end: true
+local_rank: 0
+log_level: debug
+log_level_replica: warning
+log_on_each_node: true
+logging_dir: /scratch/camembertv2/runs/results/fquad/camembertav2-base-bf16-p2-17000/max_seq_length-896-doc_stride-128-max_answer_length-30-gradient_accumulation_steps-2-precision-fp32-learning_rate-3e-05-epochs-6-lr_scheduler-cosine-warmup_steps-0/SEED-1/logs
+logging_first_step: false
+logging_nan_inf_filter: true
+logging_steps: 100
+logging_strategy: steps
+lr_scheduler_kwargs: '{}'
+lr_scheduler_type: cosine
+max_grad_norm: 1.0
+max_steps: -1
+metric_for_best_model: exact_match
+mp_parameters: .nan
+name: camembertv2/runs/results/fquad/camembertav2-base-bf16-p2-17000/max_seq_length-896-doc_stride-128-max_answer_length-30-gradient_accumulation_steps-2-precision-fp32-learning_rate-3e-05-epochs-6-lr_scheduler-cosine-warmup_steps-0
+neftune_noise_alpha: .nan
+no_cuda: false
+num_train_epochs: 6.0
+optim: adamw_torch
+optim_args: .nan
+optim_target_modules: .nan
+output_dir: /scratch/camembertv2/runs/results/fquad/camembertav2-base-bf16-p2-17000/max_seq_length-896-doc_stride-128-max_answer_length-30-gradient_accumulation_steps-2-precision-fp32-learning_rate-3e-05-epochs-6-lr_scheduler-cosine-warmup_steps-0/SEED-1
+overwrite_output_dir: false
+past_index: -1
+per_device_eval_batch_size: 64
+per_device_train_batch_size: 8
+per_gpu_eval_batch_size: .nan
+per_gpu_train_batch_size: .nan
+prediction_loss_only: false
+push_to_hub: false
+push_to_hub_model_id: .nan
+push_to_hub_organization: .nan
+push_to_hub_token: <PUSH_TO_HUB_TOKEN>
+ray_scope: last
+remove_unused_columns: true
+report_to: '[''tensorboard'']'
+restore_callback_states_from_checkpoint: false
+resume_from_checkpoint: .nan
+run_name: camembertav2-base-bf16-p2-17000
+save_on_each_node: false
+save_only_model: false
+save_safetensors: true
+save_steps: 500
+save_strategy: epoch
+save_total_limit: .nan
+seed: 1
+skip_memory_metrics: true
+split_batches: .nan
+tf32: .nan
+torch_compile: true
+torch_compile_backend: inductor
+torch_compile_mode: .nan
+torch_empty_cache_steps: .nan
+torchdynamo: .nan
+total_flos: 2.0394634246921464e+16
+tpu_metrics_debug: false
+tpu_num_cores: .nan
+train_loss: 0.5145930189164087
+train_runtime: 3736.1381
+train_samples: 20731
+train_samples_per_second: 33.293
+train_steps_per_second: 2.081
+use_cpu: false
+use_ipex: false
+use_legacy_prediction_loop: false
+use_mps_device: false
+warmup_ratio: 0.0
+warmup_steps: 0
+weight_decay: 0.0
+
+```
+
+#### Results
+
+**F1-Score:** 83.36016
+
+## Technical Specifications
+
+### Model Architecture and Objective
+
+deberta-v2 for extractive question answering in French.
+
+## Citation
+
+**BibTeX:**
+
+```bibtex
+@misc{antoun2024camembert20smarterfrench,
+      title={CamemBERT 2.0: A Smarter French Language Model Aged to Perfection},
+      author={Wissam Antoun and Francis Kulumba and Rian Touchent and Éric de la Clergerie and Benoît Sagot and Djamé Seddah},
+      year={2024},
+      eprint={2411.08868},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2411.08868},
+}
+```

all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 6.0,
+    "eval_exact_match": 64.42910915934755,
+    "eval_f1": 83.36016013340664,
+    "eval_runtime": 45.7589,
+    "eval_samples": 3188,
+    "eval_samples_per_second": 69.669,
+    "eval_steps_per_second": 1.093,
+    "total_flos": 2.0394634246921464e+16,
+    "train_loss": 0.5145930189164087,
+    "train_runtime": 3736.1381,
+    "train_samples": 20731,
+    "train_samples_per_second": 33.293,
+    "train_steps_per_second": 2.081
+}

config.json

@@ -0,0 +1,41 @@
+{
+  "_name_or_path": "/scratch/camembertv2/runs/models/camembertav2-base-bf16/post/ckpt-p2-17000/pt/discriminator/",
+  "architectures": [
+    "DebertaV2ForQuestionAnswering"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 1,
+  "conv_act": "gelu",
+  "conv_kernel_size": 0,
+  "embedding_size": 768,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 1024,
+  "max_relative_positions": -1,
+  "model_name": "camembertav2-base-bf16",
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "type_vocab_size": 0,
+  "vocab_size": 32768
+}

eval_nbest_predictions.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ef22d30d9113db2a3e121ec66eb96ad133d2c57666af0ca02aa148b9e20860b
+size 14616776

eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 6.0,
+    "eval_exact_match": 64.42910915934755,
+    "eval_f1": 83.36016013340664,
+    "eval_runtime": 45.7589,
+    "eval_samples": 3188,
+    "eval_samples_per_second": 69.669,
+    "eval_steps_per_second": 1.093
+}

logs/events.out.tfevents.1724425624.nefgpu51.72711.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f16d7346cf66e52238b68ca67654b64e55c5384a67ee75a2fefc509f7bd96a72
+size 24280

logs/events.out.tfevents.1724429414.nefgpu51.72711.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fcae142b47db7158bb33c13bc846063082e5c1e8193efeae1753e0d3c599a3a
+size 364

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a7f916f253e3a2d4b6c6311e7ac4f0137711af98793abeb56a7ed77d6f88419
+size 442496848

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,57 @@
+{
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "errors": "replace",
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": "[UNK]"
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 6.0,
+    "total_flos": 2.0394634246921464e+16,
+    "train_loss": 0.5145930189164087,
+    "train_runtime": 3736.1381,
+    "train_samples": 20731,
+    "train_samples_per_second": 33.293,
+    "train_steps_per_second": 2.081
+}

trainer_state.json

@@ -0,0 +1,635 @@
+{
+  "best_metric": 64.42910915934755,
+  "best_model_checkpoint": "/scratch/camembertv2/runs/results/fquad/camembertav2-base-bf16-p2-17000/max_seq_length-896-doc_stride-128-max_answer_length-30-gradient_accumulation_steps-2-precision-fp32-learning_rate-3e-05-epochs-6-lr_scheduler-cosine-warmup_steps-0/SEED-1/checkpoint-6480",
+  "epoch": 6.0,
+  "eval_steps": 500,
+  "global_step": 7776,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.07716049382716049,
+      "grad_norm": 23.665807723999023,
+      "learning_rate": 2.998775977415799e-05,
+      "loss": 3.705,
+      "step": 100
+    },
+    {
+      "epoch": 0.15432098765432098,
+      "grad_norm": 20.01573944091797,
+      "learning_rate": 2.9951059073049117e-05,
+      "loss": 1.6565,
+      "step": 200
+    },
+    {
+      "epoch": 0.23148148148148148,
+      "grad_norm": 12.386185646057129,
+      "learning_rate": 2.988995779332273e-05,
+      "loss": 1.4648,
+      "step": 300
+    },
+    {
+      "epoch": 0.30864197530864196,
+      "grad_norm": 18.621143341064453,
+      "learning_rate": 2.980455565410724e-05,
+      "loss": 1.243,
+      "step": 400
+    },
+    {
+      "epoch": 0.38580246913580246,
+      "grad_norm": 15.412405967712402,
+      "learning_rate": 2.96949920342655e-05,
+      "loss": 1.1382,
+      "step": 500
+    },
+    {
+      "epoch": 0.46296296296296297,
+      "grad_norm": 18.231019973754883,
+      "learning_rate": 2.95614457449243e-05,
+      "loss": 1.1534,
+      "step": 600
+    },
+    {
+      "epoch": 0.5401234567901234,
+      "grad_norm": 16.341306686401367,
+      "learning_rate": 2.940413473764923e-05,
+      "loss": 1.0392,
+      "step": 700
+    },
+    {
+      "epoch": 0.6172839506172839,
+      "grad_norm": 12.782474517822266,
+      "learning_rate": 2.9223315748741146e-05,
+      "loss": 1.0394,
+      "step": 800
+    },
+    {
+      "epoch": 0.6944444444444444,
+      "grad_norm": 15.871848106384277,
+      "learning_rate": 2.9019283880234828e-05,
+      "loss": 0.9878,
+      "step": 900
+    },
+    {
+      "epoch": 0.7716049382716049,
+      "grad_norm": 8.874638557434082,
+      "learning_rate": 2.879237211828353e-05,
+      "loss": 1.0025,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8487654320987654,
+      "grad_norm": 19.52781867980957,
+      "learning_rate": 2.8542950789715587e-05,
+      "loss": 0.9396,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9259259259259259,
+      "grad_norm": 11.584712982177734,
+      "learning_rate": 2.8271426957649866e-05,
+      "loss": 0.9625,
+      "step": 1200
+    },
+    {
+      "epoch": 1.0,
+      "eval_exact_match": 61.91969887076537,
+      "eval_f1": 81.01142429829103,
+      "eval_runtime": 46.1272,
+      "eval_samples_per_second": 69.113,
+      "eval_steps_per_second": 1.084,
+      "step": 1296
+    },
+    {
+      "epoch": 1.0030864197530864,
+      "grad_norm": 7.031412601470947,
+      "learning_rate": 2.7978243757156497e-05,
+      "loss": 0.9411,
+      "step": 1300
+    },
+    {
+      "epoch": 1.0802469135802468,
+      "grad_norm": 11.493968963623047,
+      "learning_rate": 2.7663879672047095e-05,
+      "loss": 0.6781,
+      "step": 1400
+    },
+    {
+      "epoch": 1.1574074074074074,
+      "grad_norm": 5.261780738830566,
+      "learning_rate": 2.732884775397477e-05,
+      "loss": 0.6742,
+      "step": 1500
+    },
+    {
+      "epoch": 1.2345679012345678,
+      "grad_norm": 13.310417175292969,
+      "learning_rate": 2.6973694785118392e-05,
+      "loss": 0.7004,
+      "step": 1600
+    },
+    {
+      "epoch": 1.3117283950617284,
+      "grad_norm": 17.390798568725586,
+      "learning_rate": 2.65990003858176e-05,
+      "loss": 0.6512,
+      "step": 1700
+    },
+    {
+      "epoch": 1.3888888888888888,
+      "grad_norm": 14.149763107299805,
+      "learning_rate": 2.620537606861494e-05,
+      "loss": 0.7074,
+      "step": 1800
+    },
+    {
+      "epoch": 1.4660493827160495,
+      "grad_norm": 5.80026912689209,
+      "learning_rate": 2.5793464240249014e-05,
+      "loss": 0.6629,
+      "step": 1900
+    },
+    {
+      "epoch": 1.5432098765432098,
+      "grad_norm": 17.484006881713867,
+      "learning_rate": 2.536393715322732e-05,
+      "loss": 0.6993,
+      "step": 2000
+    },
+    {
+      "epoch": 1.6203703703703702,
+      "grad_norm": 21.80438804626465,
+      "learning_rate": 2.49174958086899e-05,
+      "loss": 0.6408,
+      "step": 2100
+    },
+    {
+      "epoch": 1.6975308641975309,
+      "grad_norm": 19.416994094848633,
+      "learning_rate": 2.4454868812354406e-05,
+      "loss": 0.6592,
+      "step": 2200
+    },
+    {
+      "epoch": 1.7746913580246915,
+      "grad_norm": 5.628990650177002,
+      "learning_rate": 2.3976811185409607e-05,
+      "loss": 0.622,
+      "step": 2300
+    },
+    {
+      "epoch": 1.8518518518518519,
+      "grad_norm": 14.69774341583252,
+      "learning_rate": 2.3484103132298082e-05,
+      "loss": 0.645,
+      "step": 2400
+    },
+    {
+      "epoch": 1.9290123456790123,
+      "grad_norm": 13.895576477050781,
+      "learning_rate": 2.297754876739905e-05,
+      "loss": 0.6746,
+      "step": 2500
+    },
+    {
+      "epoch": 2.0,
+      "eval_exact_match": 63.26850690087829,
+      "eval_f1": 82.69346269167056,
+      "eval_runtime": 45.8773,
+      "eval_samples_per_second": 69.49,
+      "eval_steps_per_second": 1.09,
+      "step": 2592
+    },
+    {
+      "epoch": 2.006172839506173,
+      "grad_norm": 7.386751651763916,
+      "learning_rate": 2.2457974802689542e-05,
+      "loss": 0.6472,
+      "step": 2600
+    },
+    {
+      "epoch": 2.0833333333333335,
+      "grad_norm": 12.677722930908203,
+      "learning_rate": 2.192622919852551e-05,
+      "loss": 0.4365,
+      "step": 2700
+    },
+    {
+      "epoch": 2.1604938271604937,
+      "grad_norm": 7.171934127807617,
+      "learning_rate": 2.138317977974501e-05,
+      "loss": 0.4287,
+      "step": 2800
+    },
+    {
+      "epoch": 2.2376543209876543,
+      "grad_norm": 10.016780853271484,
+      "learning_rate": 2.082971281935195e-05,
+      "loss": 0.4462,
+      "step": 2900
+    },
+    {
+      "epoch": 2.314814814814815,
+      "grad_norm": 21.301910400390625,
+      "learning_rate": 2.0266731592091834e-05,
+      "loss": 0.4425,
+      "step": 3000
+    },
+    {
+      "epoch": 2.3919753086419755,
+      "grad_norm": 21.78326988220215,
+      "learning_rate": 1.969515490028019e-05,
+      "loss": 0.425,
+      "step": 3100
+    },
+    {
+      "epoch": 2.4691358024691357,
+      "grad_norm": 17.772539138793945,
+      "learning_rate": 1.9115915574289523e-05,
+      "loss": 0.4181,
+      "step": 3200
+    },
+    {
+      "epoch": 2.5462962962962963,
+      "grad_norm": 7.547439098358154,
+      "learning_rate": 1.8529958950142064e-05,
+      "loss": 0.4233,
+      "step": 3300
+    },
+    {
+      "epoch": 2.623456790123457,
+      "grad_norm": 9.031538963317871,
+      "learning_rate": 1.7938241326692906e-05,
+      "loss": 0.4691,
+      "step": 3400
+    },
+    {
+      "epoch": 2.700617283950617,
+      "grad_norm": 9.722735404968262,
+      "learning_rate": 1.734172840492147e-05,
+      "loss": 0.4498,
+      "step": 3500
+    },
+    {
+      "epoch": 2.7777777777777777,
+      "grad_norm": 9.985281944274902,
+      "learning_rate": 1.6741393711878455e-05,
+      "loss": 0.4388,
+      "step": 3600
+    },
+    {
+      "epoch": 2.8549382716049383,
+      "grad_norm": 9.514204978942871,
+      "learning_rate": 1.6138217011860335e-05,
+      "loss": 0.4501,
+      "step": 3700
+    },
+    {
+      "epoch": 2.932098765432099,
+      "grad_norm": 16.88687515258789,
+      "learning_rate": 1.5533182707404563e-05,
+      "loss": 0.4172,
+      "step": 3800
+    },
+    {
+      "epoch": 3.0,
+      "eval_exact_match": 63.676286072772896,
+      "eval_f1": 82.60726439387956,
+      "eval_runtime": 45.9653,
+      "eval_samples_per_second": 69.357,
+      "eval_steps_per_second": 1.088,
+      "step": 3888
+    },
+    {
+      "epoch": 3.009259259259259,
+      "grad_norm": 4.132925033569336,
+      "learning_rate": 1.4927278232714974e-05,
+      "loss": 0.3689,
+      "step": 3900
+    },
+    {
+      "epoch": 3.0864197530864197,
+      "grad_norm": 9.779620170593262,
+      "learning_rate": 1.4321492442139406e-05,
+      "loss": 0.2905,
+      "step": 4000
+    },
+    {
+      "epoch": 3.1635802469135803,
+      "grad_norm": 7.350837230682373,
+      "learning_rate": 1.371681399632967e-05,
+      "loss": 0.2937,
+      "step": 4100
+    },
+    {
+      "epoch": 3.240740740740741,
+      "grad_norm": 6.923620223999023,
+      "learning_rate": 1.3114229748717562e-05,
+      "loss": 0.2922,
+      "step": 4200
+    },
+    {
+      "epoch": 3.317901234567901,
+      "grad_norm": 16.84642791748047,
+      "learning_rate": 1.2514723134940363e-05,
+      "loss": 0.28,
+      "step": 4300
+    },
+    {
+      "epoch": 3.3950617283950617,
+      "grad_norm": 22.180021286010742,
+      "learning_rate": 1.191927256784427e-05,
+      "loss": 0.2907,
+      "step": 4400
+    },
+    {
+      "epoch": 3.4722222222222223,
+      "grad_norm": 2.5661354064941406,
+      "learning_rate": 1.1328849840685143e-05,
+      "loss": 0.2806,
+      "step": 4500
+    },
+    {
+      "epoch": 3.549382716049383,
+      "grad_norm": 11.584675788879395,
+      "learning_rate": 1.0744418541132676e-05,
+      "loss": 0.2963,
+      "step": 4600
+    },
+    {
+      "epoch": 3.626543209876543,
+      "grad_norm": 5.476423740386963,
+      "learning_rate": 1.0166932478666293e-05,
+      "loss": 0.3199,
+      "step": 4700
+    },
+    {
+      "epoch": 3.7037037037037037,
+      "grad_norm": 9.405366897583008,
+      "learning_rate": 9.597334127929346e-06,
+      "loss": 0.3107,
+      "step": 4800
+    },
+    {
+      "epoch": 3.7808641975308643,
+      "grad_norm": 8.880900382995605,
+      "learning_rate": 9.036553090582144e-06,
+      "loss": 0.2991,
+      "step": 4900
+    },
+    {
+      "epoch": 3.8580246913580245,
+      "grad_norm": 3.8892629146575928,
+      "learning_rate": 8.485504578164017e-06,
+      "loss": 0.2716,
+      "step": 5000
+    },
+    {
+      "epoch": 3.935185185185185,
+      "grad_norm": 5.704967498779297,
+      "learning_rate": 7.945087918440563e-06,
+      "loss": 0.2688,
+      "step": 5100
+    },
+    {
+      "epoch": 4.0,
+      "eval_exact_match": 64.2409033877039,
+      "eval_f1": 83.135484930466,
+      "eval_runtime": 45.9298,
+      "eval_samples_per_second": 69.41,
+      "eval_steps_per_second": 1.089,
+      "step": 5184
+    },
+    {
+      "epoch": 4.012345679012346,
+      "grad_norm": 16.147579193115234,
+      "learning_rate": 7.416185087673616e-06,
+      "loss": 0.2919,
+      "step": 5200
+    },
+    {
+      "epoch": 4.089506172839506,
+      "grad_norm": 13.380005836486816,
+      "learning_rate": 6.899659271209459e-06,
+      "loss": 0.2068,
+      "step": 5300
+    },
+    {
+      "epoch": 4.166666666666667,
+      "grad_norm": 9.491084098815918,
+      "learning_rate": 6.3963534547343126e-06,
+      "loss": 0.2009,
+      "step": 5400
+    },
+    {
+      "epoch": 4.243827160493828,
+      "grad_norm": 14.11040210723877,
+      "learning_rate": 5.907089048496351e-06,
+      "loss": 0.2124,
+      "step": 5500
+    },
+    {
+      "epoch": 4.320987654320987,
+      "grad_norm": 12.674304962158203,
+      "learning_rate": 5.4326645467394085e-06,
+      "loss": 0.2173,
+      "step": 5600
+    },
+    {
+      "epoch": 4.398148148148148,
+      "grad_norm": 5.682621955871582,
+      "learning_rate": 4.973854224536363e-06,
+      "loss": 0.213,
+      "step": 5700
+    },
+    {
+      "epoch": 4.4753086419753085,
+      "grad_norm": 5.133475303649902,
+      "learning_rate": 4.5314068741488615e-06,
+      "loss": 0.2,
+      "step": 5800
+    },
+    {
+      "epoch": 4.552469135802469,
+      "grad_norm": 6.370384693145752,
+      "learning_rate": 4.1060445829758305e-06,
+      "loss": 0.197,
+      "step": 5900
+    },
+    {
+      "epoch": 4.62962962962963,
+      "grad_norm": 16.37765884399414,
+      "learning_rate": 3.6984615550850894e-06,
+      "loss": 0.2051,
+      "step": 6000
+    },
+    {
+      "epoch": 4.70679012345679,
+      "grad_norm": 11.54761791229248,
+      "learning_rate": 3.3093229782514023e-06,
+      "loss": 0.1733,
+      "step": 6100
+    },
+    {
+      "epoch": 4.783950617283951,
+      "grad_norm": 22.175281524658203,
+      "learning_rate": 2.939263938350012e-06,
+      "loss": 0.2003,
+      "step": 6200
+    },
+    {
+      "epoch": 4.861111111111111,
+      "grad_norm": 1.2753137350082397,
+      "learning_rate": 2.588888382877342e-06,
+      "loss": 0.194,
+      "step": 6300
+    },
+    {
+      "epoch": 4.938271604938271,
+      "grad_norm": 32.319236755371094,
+      "learning_rate": 2.2587681352905404e-06,
+      "loss": 0.2149,
+      "step": 6400
+    },
+    {
+      "epoch": 5.0,
+      "eval_exact_match": 64.42910915934755,
+      "eval_f1": 83.36016013340664,
+      "eval_runtime": 45.8927,
+      "eval_samples_per_second": 69.466,
+      "eval_steps_per_second": 1.089,
+      "step": 6480
+    },
+    {
+      "epoch": 5.015432098765432,
+      "grad_norm": 9.053484916687012,
+      "learning_rate": 1.9494419617743312e-06,
+      "loss": 0.198,
+      "step": 6500
+    },
+    {
+      "epoch": 5.092592592592593,
+      "grad_norm": 11.60733699798584,
+      "learning_rate": 1.6614146919584094e-06,
+      "loss": 0.1512,
+      "step": 6600
+    },
+    {
+      "epoch": 5.169753086419753,
+      "grad_norm": 9.327279090881348,
+      "learning_rate": 1.3951563950202656e-06,
+      "loss": 0.167,
+      "step": 6700
+    },
+    {
+      "epoch": 5.246913580246914,
+      "grad_norm": 4.551391124725342,
+      "learning_rate": 1.1511016125181445e-06,
+      "loss": 0.1315,
+      "step": 6800
+    },
+    {
+      "epoch": 5.324074074074074,
+      "grad_norm": 4.2411274909973145,
+      "learning_rate": 9.296486492061334e-07,
+      "loss": 0.1532,
+      "step": 6900
+    },
+    {
+      "epoch": 5.401234567901234,
+      "grad_norm": 5.1193461418151855,
+      "learning_rate": 7.311589229888083e-07,
+      "loss": 0.1624,
+      "step": 7000
+    },
+    {
+      "epoch": 5.478395061728395,
+      "grad_norm": 1.735378384590149,
+      "learning_rate": 5.55956375076332e-07,
+      "loss": 0.1688,
+      "step": 7100
+    },
+    {
+      "epoch": 5.555555555555555,
+      "grad_norm": 3.7359869480133057,
+      "learning_rate": 4.043269413026429e-07,
+      "loss": 0.148,
+      "step": 7200
+    },
+    {
+      "epoch": 5.632716049382716,
+      "grad_norm": 6.915912628173828,
+      "learning_rate": 2.7651808546956646e-07,
+      "loss": 0.1822,
+      "step": 7300
+    },
+    {
+      "epoch": 5.709876543209877,
+      "grad_norm": 5.2128448486328125,
+      "learning_rate": 1.727383954784373e-07,
+      "loss": 0.163,
+      "step": 7400
+    },
+    {
+      "epoch": 5.787037037037037,
+      "grad_norm": 3.4656639099121094,
+      "learning_rate": 9.315724290836047e-08,
+      "loss": 0.1716,
+      "step": 7500
+    },
+    {
+      "epoch": 5.864197530864198,
+      "grad_norm": 21.396251678466797,
+      "learning_rate": 3.790450659670097e-08,
+      "loss": 0.1694,
+      "step": 7600
+    },
+    {
+      "epoch": 5.9413580246913575,
+      "grad_norm": 14.200843811035156,
+      "learning_rate": 7.070360672907228e-09,
+      "loss": 0.1618,
+      "step": 7700
+    },
+    {
+      "epoch": 6.0,
+      "eval_exact_match": 64.0840652446675,
+      "eval_f1": 83.12314115625247,
+      "eval_runtime": 45.7565,
+      "eval_samples_per_second": 69.673,
+      "eval_steps_per_second": 1.093,
+      "step": 7776
+    },
+    {
+      "epoch": 6.0,
+      "step": 7776,
+      "total_flos": 2.0394634246921464e+16,
+      "train_loss": 0.5145930189164087,
+      "train_runtime": 3736.1381,
+      "train_samples_per_second": 33.293,
+      "train_steps_per_second": 2.081
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 7776,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.0394634246921464e+16,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b9787a37b5a87abea534c694a63b96bc13b2ab599a8e1ee426033703ba558db
+size 5688