| train_input: | |
| batch_size: 256 | |
| data_processor: GptHDF5MapDataProcessor | |
| data_dir: /cra-406/datasets/jais_instruction_datasets/v12p2/tokenized_mlv2_2k/ | |
| num_workers: 1 | |
| persistent_workers: true | |
| prefetch_factor: 10 | |
| repeat: true | |
| shuffle: false | |
| shuffle_seed: 1 | |
| use_worker_cache: false | |
| vocab_size: 84992 | |
| eval_input: | |
| batch_size: 32 | |
| data_dir: /cb/customers/g42/datasets/multilingual_v2/pile_val_correct_packed | |
| data_processor: GptHDF5MapDataProcessor | |
| num_workers: 1 | |
| repeat: false | |
| shuffle: false | |
| use_worker_cache: false | |
| vocab_size: 84992 | |
| model: | |
| mixed_precision: true | |
| fp16_type: cbfloat16 | |
| boundary_casting: false | |
| lora_params: null | |
| vocab_size: 84992 | |
| embedding_layer_norm: false | |
| embedding_dropout_rate: 0.0 | |
| share_embedding_weights: true | |
| position_embedding_type: alibi | |
| max_position_embeddings: 2048 | |
| position_embedding_offset: 0 | |
| num_relative_attention_buckets: 32 | |
| rotary_dim: null | |
| rope_theta: 10000 | |
| pad_rope: false | |
| alibi_trainable_slopes: false | |
| pos_scaling_factor: 1.0 | |
| hidden_size: 1088 | |
| num_hidden_layers: 14 | |
| dropout_rate: 0.0 | |
| norm_type: layernorm | |
| layer_norm_epsilon: 1.0e-05 | |
| num_heads: 17 | |
| attention_module: aiayn_attention | |
| extra_attention_params: {} | |
| attention_type: scaled_dot_product | |
| attention_dropout_rate: 0.0 | |
| use_projection_bias_in_attention: true | |
| use_ffn_bias_in_attention: true | |
| attention_softmax_fp32: false | |
| attention_kernel: optimized_beta | |
| attention_sliding_window_length: null | |
| scale_qk_dot_by_layer_idx: false | |
| fixed_sparse_attention: null | |
| filter_size: 2912 | |
| nonlinearity: swiglu | |
| use_ffn_bias: true | |
| use_bias_in_output: false | |
| loss_scaling: num_tokens | |
| loss_weight: 1.0 | |
| embeddings_scale: 9.1705785388303 | |
| scale_qk_dot_by_d: true | |
| output_logits_scale: 0.2576902348606329 | |
| initializer: | |
| name: truncated_normal | |
| mean: 0.0 | |
| std: 0.04203434605680388 | |
| a: -0.08406869211360776 | |
| b: 0.08406869211360776 | |
| nonlinearity: null | |
| mode: null | |
| scale: null | |
| distribution: null | |
| initializer_range: 0.02 | |
| embedding_initializer: | |
| name: truncated_normal | |
| mean: 0.0 | |
| std: 0.0866560243479838 | |
| a: -0.1733120486959676 | |
| b: 0.1733120486959676 | |
| nonlinearity: null | |
| mode: null | |
| scale: null | |
| distribution: null | |
| output_layer_initializer: | |
| name: truncated_normal | |
| mean: 0.0 | |
| std: 0.007943744727823684 | |
| a: -0.015887489455647368 | |
| b: 0.015887489455647368 | |
| nonlinearity: null | |
| mode: null | |
| scale: null | |
| distribution: null | |
| compute_eval_metrics: true | |
| sparsity: null | |
| optimizer: | |
| optimizer_type: AdamW | |
| weight_decay: 0.1 | |
| log_summaries: true | |
| loss_scaling_factor: dynamic | |
| learning_rate: | |
| - end_learning_rate: 0.0016 | |
| initial_learning_rate: 0.0 | |
| scheduler: Linear | |
| total_iters: 695 | |
| - end_learning_rate: 0.00016 | |
| initial_learning_rate: 0.0016 | |
| scheduler: Linear | |
| total_iters: 23995 | |
| max_gradient_norm: 1.0 | |
| adjust_learning_rate: | |
| decoder_kernel: 0.23529411764705882 | |
| betas: | |
| - 0.9 | |
| - 0.95 | |
| correct_bias: true | |
| eps: 1.0e-08 | |
| runconfig: | |
| steps_per_epoch: null | |
| max_steps: 24690 | |
| mgmt_address: null | |
| mount_dirs: | |
| - /cra-406 | |
| num_epochs: null | |
| python_paths: | |
| - /cra-406/workdirs/modelzoos/rel-2.2.1/modelzoo/src | |
| compile_dir: null | |
| checkpoint_path: /cra-406/workdirs/240209_Jais_series_v3/artifacts/model_dir_256M/checkpoint_240320.mdl | |
| credentials_path: null | |
| debug_args_path: null | |
| retrace_every_iteration: null | |
| eval_steps: 5219 | |
| init_method: env:// | |
| job_time_sec: null | |
| job_labels: | |
| - Name=Neha_Sengupta | |
| - Organization=Core42 | |
| - Model=Jais_256M | |
| - Mode=Train | |
| - Num_CSX=4 | |
| - Language=Bilingual | |
| - Type=Train | |
| - Dataset=v12p2 | |
| job_priority: p3 | |
| seed: 1 | |
| mgmt_namespace: null | |
| load_checkpoint_states: model | |
| target_device: CSX | |
| mode: train | |
| wsc_log_level: null | |
| autoload_last_checkpoint: true | |
| check_loss_values: true | |
| disable_strict_checkpoint_loading: null | |
| dist_addr: localhost:8888 | |
| dist_backend: nccl | |
| checkpoint_steps: 8231 | |
| disable_version_check: true | |
| drop_data: false | |
| enable_distributed: false | |
| model_dir: artifacts/jais_256M_v12p2_gbs256 | |
| save_initial_checkpoint: false | |
| precision_opt_level: 1 | |
| num_workers_per_csx: 0 | |
| validate_only: null | |
| logging: null | |
| sync_batchnorm: false | |
| compile_only: null | |
| log_steps: 1 | |
| num_steps: null | |
| transfer_processes: null | |
| num_wgt_servers: null | |
| num_csx: 4 | |
| num_act_servers: null | |
| eval_frequency: null | |
| execute_crd_memory_gi: null | |
| compile_crd_memory_gi: null | |
| op_profiler_config: null | |
| dump_activations: false | |
| log_input_summaries: false | |
| main_process_id: 0 | |
| max_checkpoints: 100000 | |
| summary_dir: null | |
| lazy_initialization: true | |
| use_cstorch_optimizer_step: false | |
| wrk_memory_gi: null | |
| act_memory_gi: null | |
| cmd_memory_gi: null | |
| wgt_memory_gi: null | |
| experimental: {} | |
| ini: null | |
| debug_args: null | |