max_seq_len: 1024
seed: 1
precision: fp32

# If you are using one model, put it here:
model_name_or_path: EleutherAI/gpt-neo-125m
# otherwise, write a block for each model you want to test in the `models` section

models:
-
  model_name: ${model_name_or_path}
  model:
    name: hf_causal_lm
    pretrained_model_name_or_path: ${model_name_or_path}
    init_device: mixed
    pretrained: true
  tokenizer:
    name: ${model_name_or_path}
    kwargs:
      model_max_length: ${max_seq_len}
# # if you are evaluating more than one model, list them all as YAML blocks without variable interpolation
# -
#   model_name: mosaicml/mpt-7b
#   model:
#     name: hf_causal_lm
#     pretrained_model_name_or_path: mosaicml/mpt-7b
#     init_device: cpu
#     pretrained: true
#     config_overrides:
#       max_seq_len: ${max_seq_len}
#   tokenizer:
#     name: mosaicml/mpt-7b
#     kwargs:
#       model_max_length: ${max_seq_len}


device_eval_batch_size: 4

# FSDP config for model sharding
# fsdp_config:
#   sharding_strategy: FULL_SHARD
#   mixed_precision: FULL
#   forward_prefetch: True
#   limit_all_gathers: True

icl_tasks: 'eval/yamls/tasks.yaml'
eval_gauntlet: 'eval/yamls/eval_gauntlet.yaml'