| train_input: | |
| batch_size: 976 | |
| data_processor: GptHDF5MapDataProcessor | |
| mixture: | |
| - data_dir: /cra-406/datasets/pile/multilingual_v2/train_correct/ | |
| weight: 0.6510508179774476 | |
| - data_dir: /cra-406/datasets/github/multilingual_v2/packed_2k/train | |
| weight: 0.055087602323960365 | |
| - data_dir: /cra-406/datasets/books3_arabic/multilingual_v2/packed_2k/books_3_arabic_train_correct_packed | |
| weight: 0.031560734650858936 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/alkhair_train_packed | |
| weight: 0.0008441127388845985 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/aranews_train_packed | |
| weight: 0.00015702987060793174 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/baai_train_packed | |
| weight: 0.02652363386071335 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/C4_train_packed | |
| weight: 0.04370135940994404 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/ccnews_train_packed | |
| weight: 0.006820988629070355 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/common_crawl_train_packed | |
| weight: 0.16413286051785408 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/daypop_train_packed | |
| weight: 0.001772579714458703 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/en2ar_wikipedia_train_packed | |
| weight: 0.006335165657431352 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/en_wikipedia_train_packed | |
| weight: 0.0035095904892209306 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/maktabah_train_packed | |
| weight: 0.002642036817637927 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/misc_train_packed | |
| weight: 6.954077746676907e-05 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/osian_train_packed | |
| weight: 0.0006243331144143421 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/un_train_packed | |
| weight: 0.001005513115682201 | |
| - data_dir: /cra-406/datasets/AraV5/multilingual_v2/packed_2k/train/wikipedia_train_packed | |
| weight: 0.00034892678537459647 | |
| - data_dir: /cra-406/datasets/multilingual_v2/uae_news_sep_2023_12b_split/ar/train_shuffled | |
| weight: 0.0012430476743474177 | |
| - data_dir: /cra-406/datasets/multilingual_v2/itc_663m_split/train_shuffled | |
| weight: 0.0013597894242614768 | |
| - data_dir: /cra-406/datasets/multilingual_v2/uae_news_sep_2023_12b_split/en/train_shuffled | |
| weight: 0.0012103364503629503 | |
| num_workers: 1 | |
| persistent_workers: true | |
| prefetch_factor: 10 | |
| repeat: true | |
| shuffle: false | |
| shuffle_seed: 1 | |
| use_worker_cache: false | |
| vocab_size: 84992 | |
| eval_input: | |
| batch_size: 32 | |
| data_dir: /cb/customers/g42/datasets/multilingual_v2/pile_val_correct_packed | |
| data_processor: GptHDF5MapDataProcessor | |
| num_workers: 1 | |
| repeat: false | |
| shuffle: false | |
| use_worker_cache: false | |
| vocab_size: 84992 | |
| model: | |
| mixed_precision: true | |
| fp16_type: cbfloat16 | |
| boundary_casting: false | |
| lora_params: null | |
| vocab_size: 84992 | |
| embedding_layer_norm: false | |
| embedding_dropout_rate: 0.0 | |
| share_embedding_weights: true | |
| position_embedding_type: alibi | |
| max_position_embeddings: 2048 | |
| position_embedding_offset: 0 | |
| num_relative_attention_buckets: 32 | |
| rotary_dim: null | |
| rope_theta: 10000 | |
| pad_rope: false | |
| alibi_trainable_slopes: false | |
| pos_scaling_factor: 1.0 | |
| hidden_size: 1088 | |
| num_hidden_layers: 14 | |
| dropout_rate: 0.0 | |
| norm_type: layernorm | |
| layer_norm_epsilon: 1.0e-05 | |
| num_heads: 17 | |
| attention_module: aiayn_attention | |
| extra_attention_params: {} | |
| attention_type: scaled_dot_product | |
| attention_dropout_rate: 0.0 | |
| use_projection_bias_in_attention: true | |
| use_ffn_bias_in_attention: true | |
| attention_softmax_fp32: false | |
| attention_kernel: optimized_beta | |
| attention_sliding_window_length: null | |
| scale_qk_dot_by_layer_idx: false | |
| fixed_sparse_attention: null | |
| filter_size: 2912 | |
| nonlinearity: swiglu | |
| use_ffn_bias: true | |
| use_bias_in_output: false | |
| loss_scaling: num_tokens | |
| loss_weight: 1.0 | |
| embeddings_scale: 9.1705785388303 | |
| scale_qk_dot_by_d: true | |
| output_logits_scale: 0.2576902348606329 | |
| initializer: | |
| name: truncated_normal | |
| mean: 0.0 | |
| std: 0.04203434605680388 | |
| a: -0.08406869211360776 | |
| b: 0.08406869211360776 | |
| nonlinearity: null | |
| mode: null | |
| scale: null | |
| distribution: null | |
| initializer_range: 0.02 | |
| embedding_initializer: | |
| name: truncated_normal | |
| mean: 0.0 | |
| std: 0.0866560243479838 | |
| a: -0.1733120486959676 | |
| b: 0.1733120486959676 | |
| nonlinearity: null | |
| mode: null | |
| scale: null | |
| distribution: null | |
| output_layer_initializer: | |
| name: truncated_normal | |
| mean: 0.0 | |
| std: 0.007943744727823684 | |
| a: -0.015887489455647368 | |
| b: 0.015887489455647368 | |
| nonlinearity: null | |
| mode: null | |
| scale: null | |
| distribution: null | |
| compute_eval_metrics: true | |
| sparsity: null | |
| optimizer: | |
| optimizer_type: AdamW | |
| weight_decay: 0.1 | |
| log_summaries: true | |
| loss_scaling_factor: dynamic | |
| learning_rate: | |
| - end_learning_rate: 0.015625 | |
| initial_learning_rate: 0.0 | |
| scheduler: Linear | |
| total_iters: 187 | |
| - end_learning_rate: 1.9196e-05 | |
| initial_learning_rate: 0.015625 | |
| scheduler: Linear | |
| total_iters: 240133 | |
| max_gradient_norm: 1.0 | |
| adjust_learning_rate: | |
| decoder_kernel: 0.23529411764705882 | |
| betas: | |
| - 0.9 | |
| - 0.95 | |
| correct_bias: true | |
| eps: 1.0e-08 | |
| runconfig: | |
| steps_per_epoch: null | |
| max_steps: 240320 | |
| mgmt_address: null | |
| mount_dirs: | |
| - /cra-406 | |
| num_epochs: null | |
| python_paths: | |
| - /cra-406/workdirs/modelzoos/rel-2.2.1/modelzoo/src | |
| compile_dir: null | |
| checkpoint_path: null | |
| credentials_path: null | |
| debug_args_path: null | |
| retrace_every_iteration: null | |
| eval_steps: 5219 | |
| init_method: env:// | |
| job_time_sec: null | |
| job_labels: | |
| - Name=Neha_Sengupta | |
| - Organization=Inception | |
| - Model=Jais_256M | |
| - Mode=Train | |
| - Num_CSX=8 | |
| - Language=Bilingual | |
| - Type=Train | |
| - Dataset=AraV5_Pile_Github_Books_UAE_ITC | |
| job_priority: p2 | |
| seed: 1 | |
| mgmt_namespace: cra-406 | |
| load_checkpoint_states: model,optimizer,global_step,dataloader,lr_scheduler | |
| target_device: CSX | |
| mode: train | |
| wsc_log_level: null | |
| autoload_last_checkpoint: true | |
| check_loss_values: true | |
| disable_strict_checkpoint_loading: null | |
| dist_addr: localhost:8888 | |
| dist_backend: nccl | |
| checkpoint_steps: 24032 | |
| disable_version_check: null | |
| drop_data: false | |
| enable_distributed: false | |
| model_dir: artifacts/model_dir_256M | |
| save_initial_checkpoint: false | |
| precision_opt_level: 1 | |
| num_workers_per_csx: 0 | |
| validate_only: null | |
| logging: null | |
| sync_batchnorm: false | |
| compile_only: null | |
| log_steps: 1 | |
| num_steps: null | |
| transfer_processes: null | |
| num_wgt_servers: null | |
| num_csx: 8 | |
| num_act_servers: null | |
| eval_frequency: null | |
| execute_crd_memory_gi: null | |
| compile_crd_memory_gi: null | |
| op_profiler_config: null | |
| dump_activations: false | |
| log_input_summaries: false | |
| main_process_id: 0 | |
| max_checkpoints: 100000 | |
| summary_dir: null | |
| lazy_initialization: true | |
| use_cstorch_optimizer_step: false | |
| wrk_memory_gi: null | |
| act_memory_gi: null | |
| cmd_memory_gi: null | |
| wgt_memory_gi: null | |
| experimental: {} | |
| ini: | |
| ws_opt_speculate_optimizer: true | |
| debug_args: null | |