Text Generation
Transformers
Safetensors
English
mistral
conversational
text-generation-inference
Inference Endpoints
winglian commited on
Commit
a09347d
1 Parent(s): 64ae0db

Create configs/axolotl.yml

Browse files
Files changed (1) hide show
  1. configs/axolotl.yml +88 -0
configs/axolotl.yml ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ base_model: teknium/OpenHermes-2.5-Mistral-7B
2
+ model_type: MistralForCausalLM
3
+ tokenizer_type: LlamaTokenizer
4
+ is_mistral_derived_model: true
5
+
6
+ load_in_8bit: false
7
+ load_in_4bit: false
8
+ strict: false
9
+
10
+ rl: true
11
+ datasets:
12
+ - path: Intel/orca_dpo_pairs
13
+ split: train
14
+ type: intel_apply_chatml
15
+ - path: allenai/ultrafeedback_binarized_cleaned
16
+ split: train_prefs
17
+ type: ultra_apply_chatml
18
+ dataset_prepared_path: last_run_prepared
19
+ val_set_size: 0.0
20
+ output_dir: ./dpopenhermes-rc5/
21
+ save_total_limit: 3
22
+ hub_model_id: openaccess-ai-collective/dpopenhermes-rc5
23
+
24
+ adapter: lora
25
+ lora_model_dir:
26
+
27
+ sequence_len: 2048
28
+ sample_packing: false
29
+ pad_to_sequence_len: false
30
+
31
+ lora_r: 64
32
+ lora_alpha: 32
33
+ lora_dropout: 0.05
34
+ lora_target_linear: true
35
+ lora_modules_to_save:
36
+ - embed_tokens
37
+ - lm_head
38
+ lora_fan_in_fan_out:
39
+ lora_target_modules:
40
+ - gate_proj
41
+ - down_proj
42
+ - up_proj
43
+ - q_proj
44
+ - v_proj
45
+ - k_proj
46
+ - o_proj
47
+
48
+ wandb_project: openhermes-dpo
49
+ wandb_entity: oaaic
50
+ wandb_watch:
51
+ wandb_run_id:
52
+ wandb_log_model:
53
+
54
+ gradient_accumulation_steps: 4
55
+ micro_batch_size: 4
56
+ num_epochs: 1
57
+ optimizer: paged_adamw_8bit
58
+ adam_beta2: 0.95
59
+ adam_epsilion: 0.00001
60
+ lr_scheduler: cosine
61
+ learning_rate: 2e-5
62
+
63
+ train_on_inputs: false
64
+ group_by_length: false
65
+ bf16: true
66
+ fp16: false
67
+ tf32: true
68
+
69
+ gradient_checkpointing: true
70
+ early_stopping_patience:
71
+ resume_from_checkpoint:
72
+ local_rank:
73
+ logging_steps: 1
74
+ xformers_attention:
75
+ flash_attention: true
76
+
77
+ warmup_steps: 100
78
+ eval_steps:
79
+ eval_table_size:
80
+ eval_table_max_new_tokens: 128
81
+ save_steps: 239
82
+ debug:
83
+ deepspeed:
84
+ weight_decay: 0.1
85
+ fsdp:
86
+ fsdp_config:
87
+ special_tokens:
88
+ save_safetensors: true