inoid commited on
Commit
97392da
·
1 Parent(s): f577be4

Add appy_chat_template process

Browse files
Files changed (1) hide show
  1. spanish_medica_llm.py +3 -0
spanish_medica_llm.py CHANGED
@@ -702,6 +702,7 @@ def run_finnetuning_process():
702
  os.environ['WANDB_DISABLED'] = 'true'
703
  tokenizer = loadSpanishTokenizer()
704
  medicalSpanishDataset = applyChatInstructFormat( loadSpanishDatasetFinnetuning())
 
705
  medicalSpanishDataset = medicalSpanishDataset.train_test_split(0.2, seed=203984)
706
  train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid( medicalSpanishDataset )
707
 
@@ -710,4 +711,6 @@ def run_finnetuning_process():
710
  print('Dataset in One ')
711
  print (train_dataset[5])
712
  configAndRunFineTuning(base_model,train_dataset, eval_dataset, tokenizer)
 
 
713
 
 
702
  os.environ['WANDB_DISABLED'] = 'true'
703
  tokenizer = loadSpanishTokenizer()
704
  medicalSpanishDataset = applyChatInstructFormat( loadSpanishDatasetFinnetuning())
705
+ medicalSpanishDataset = tokenizer.apply_chat_template(medicalSpanishDataset, tokenize=False)
706
  medicalSpanishDataset = medicalSpanishDataset.train_test_split(0.2, seed=203984)
707
  train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid( medicalSpanishDataset )
708
 
 
711
  print('Dataset in One ')
712
  print (train_dataset[5])
713
  configAndRunFineTuning(base_model,train_dataset, eval_dataset, tokenizer)
714
+ def generate_response(query):
715
+
716