Spaces:
Runtime error
Runtime error
Add appy_chat_template process
Browse files- spanish_medica_llm.py +3 -0
spanish_medica_llm.py
CHANGED
@@ -702,6 +702,7 @@ def run_finnetuning_process():
|
|
702 |
os.environ['WANDB_DISABLED'] = 'true'
|
703 |
tokenizer = loadSpanishTokenizer()
|
704 |
medicalSpanishDataset = applyChatInstructFormat( loadSpanishDatasetFinnetuning())
|
|
|
705 |
medicalSpanishDataset = medicalSpanishDataset.train_test_split(0.2, seed=203984)
|
706 |
train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid( medicalSpanishDataset )
|
707 |
|
@@ -710,4 +711,6 @@ def run_finnetuning_process():
|
|
710 |
print('Dataset in One ')
|
711 |
print (train_dataset[5])
|
712 |
configAndRunFineTuning(base_model,train_dataset, eval_dataset, tokenizer)
|
|
|
|
|
713 |
|
|
|
702 |
os.environ['WANDB_DISABLED'] = 'true'
|
703 |
tokenizer = loadSpanishTokenizer()
|
704 |
medicalSpanishDataset = applyChatInstructFormat( loadSpanishDatasetFinnetuning())
|
705 |
+
medicalSpanishDataset = tokenizer.apply_chat_template(medicalSpanishDataset, tokenize=False)
|
706 |
medicalSpanishDataset = medicalSpanishDataset.train_test_split(0.2, seed=203984)
|
707 |
train_dataset, eval_dataset, test_dataset = splitDatasetInTestValid( medicalSpanishDataset )
|
708 |
|
|
|
711 |
print('Dataset in One ')
|
712 |
print (train_dataset[5])
|
713 |
configAndRunFineTuning(base_model,train_dataset, eval_dataset, tokenizer)
|
714 |
+
def generate_response(query):
|
715 |
+
|
716 |
|