Spaces:
Sleeping
Sleeping
File size: 2,600 Bytes
17ff0d8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 |
PYTHON_CMD="
accelerate launch
--mixed_precision bf16 -m sdlm.run_pretrain \
--per_device_train_batch_size 1 \
--per_device_eval_batch_size 1 \
--do_train \
--do_eval \
--log_level info \
--evaluation_strategy steps \
--report_to tensorboard \
--max_seq_length 2048 \
--simplex_value 5 \
--num_diffusion_steps 5000 \
--lr_scheduler_type constant_with_warmup \
--learning_rate 1e-5 \
--pad_to_max_length \
--beta_schedule squaredcos_improved_ddpm \
--top_p 0.99 \
--max_steps 200000 \
--warmup_steps 5000 \
--logging_steps 50 \
--save_total_limit 1 \
--conditional_generation ul2 \
--self_condition logits_mean \
--self_condition_mix_before_weights \
--streaming \
--bf16 \
--optim adamw_torch_fused \
--gradient_checkpointing \
--use_flash_attention2 \
--is_causal false \
--mask_padding_in_loss false \
--without_compute_metrics true \
--dataloader_num_workers 8 \
--remove_unused_columns false \
--dispatch_batches false \
--shuffle true \
--preprocessing_num_workers 16 \
--line_by_line false \
--model_revision 26bca36bde8333b5d7f72e9ed20ccda6a618af24 \
"
if [ ! -z "${BEAKER}" ]; then
GANTRY_CMD="gantry run -y -n mistral_pretrained -t mistral_pretrained --allow-dirty \
--workspace ai2/tess2 \
--gpus 7 \
--priority normal \
--budget ai2/allennlp \
--preemptible \
--env PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python \
--env-secret HF_TOKEN=HF_TOKEN \
--beaker-image ai2/pytorch2.0.0-cuda11.8-python3.10 \
--venv base \
--pip requirements.txt \
--no-nfs \
--cluster ai2/jupiter-cirrascale-2 \
--weka oe-data-default:/data/input \
-- ${PYTHON_CMD} \
--model_name_or_path mistralai/Mistral-7B-v0.1 \
--eval_steps 2000 \
--save_steps 2000 \
--max_eval_samples 200 \
--gradient_accumulation_steps 16 \
--num_inference_diffusion_steps 100 \
--overwrite_output_dir false \
--beaker \
--output_dir /results \
"
else
${PYTHON_CMD} \
--model_name_or_path mistralai/Mistral-7B-v0.1 \
--eval_steps 2000 \
--save_steps 2000 \
--max_eval_samples 16 \
--gradient_accumulation_steps 1 \
--num_inference_diffusion_steps 100 \
--output_dir pretrained_tess_model \
--overwrite_output_dir true
fi
|