Perceptrix/finetune/scripts/eval/yamls/lm_tasks.yaml

icl_tasks:
-
  label: jeopardy
  dataset_uri: eval/local_data/world_knowledge/jeopardy_all.jsonl # ADD YOUR OWN DATASET URI
  num_fewshot: [10]
  icl_task_type: language_modeling
  continuation_delimiter: "\nAnswer: " # this separates questions from answers
  has_categories: true
-
  label: bigbench_qa_wikidata
  dataset_uri: eval/local_data/world_knowledge/bigbench_qa_wikidata.jsonl # ADD YOUR OWN DATASET URI
  num_fewshot: [10]
  icl_task_type: language_modeling
-
  label: lambada_openai
  dataset_uri: eval/local_data/language_understanding/lambada_openai.jsonl
  num_fewshot: [0]
  icl_task_type: language_modeling
-
  label: bigbench_conlang_translation
  dataset_uri: eval/local_data/language_understanding/bigbench_conlang_translation.jsonl
  num_fewshot: [0]
  icl_task_type: language_modeling
-
  label: bigbench_dyck_languages
  dataset_uri: eval/local_data/symbolic_problem_solving/bigbench_dyck_languages.jsonl
  num_fewshot: [10]
  icl_task_type: language_modeling
-
  label: bigbench_cs_algorithms
  dataset_uri: eval/local_data/symbolic_problem_solving/bigbench_cs_algorithms.jsonl
  num_fewshot: [10]
  icl_task_type: language_modeling
-
  label: bigbench_operators
  dataset_uri: eval/local_data/symbolic_problem_solving/bigbench_operators.jsonl
  num_fewshot: [10]
  icl_task_type: language_modeling
-
  label: bigbench_repeat_copy_logic
  dataset_uri: eval/local_data/symbolic_problem_solving/bigbench_repeat_copy_logic.jsonl
  num_fewshot: [10]
  icl_task_type: language_modeling
-
  label: simple_arithmetic_nospaces
  dataset_uri: eval/local_data/symbolic_problem_solving/simple_arithmetic_nospaces.jsonl
  num_fewshot: [10]
  icl_task_type: language_modeling
-
  label: simple_arithmetic_withspaces
  dataset_uri: eval/local_data/symbolic_problem_solving/simple_arithmetic_withspaces.jsonl
  num_fewshot: [10]
  icl_task_type: language_modeling
-
  label: pubmed_qa_labeled
  dataset_uri: eval/local_data/reading_comprehension/pubmed_qa_labeled.jsonl # ADD YOUR OWN DATASET URI
  num_fewshot: [10]
  icl_task_type: language_modeling
-
  label: squad
  dataset_uri: eval/local_data/reading_comprehension/squad.jsonl # ADD YOUR OWN DATASET URI
  num_fewshot: [10]
  icl_task_type: language_modeling