darpanaswal commited on
Commit
699e2de
·
verified ·
1 Parent(s): 8ed9db0

Update finetune.py

Browse files
Files changed (1) hide show
  1. finetune.py +4 -4
finetune.py CHANGED
@@ -89,12 +89,12 @@ def fine_tune(model_name, finetune_type, model, tokenizer, summarize_text, train
89
  max_output = 60
90
  train_dataset = train
91
  eval_dataset = val
92
- if finetune_type == "multilingual":
93
- train_dataset = Dataset.from_pandas(train.sample(1200))
94
- eval_dataset = Dataset.from_pandas(val.sample(150))
95
- else:
96
  train_dataset = Dataset.from_pandas(train.sample(1500))
97
  eval_dataset = Dataset.from_pandas(val.sample(200))
 
 
 
98
  def preprocess_function(examples):
99
  inputs = [f"Summarize the text: {ex}" for ex in examples["source"]]
100
  targets = [f"Summary: {ex}" for ex in examples["target"]]
 
89
  max_output = 60
90
  train_dataset = train
91
  eval_dataset = val
92
+ if finetune_type == "english":
 
 
 
93
  train_dataset = Dataset.from_pandas(train.sample(1500))
94
  eval_dataset = Dataset.from_pandas(val.sample(200))
95
+ else:
96
+ train_dataset = Dataset.from_pandas(train.sample(1200))
97
+ eval_dataset = Dataset.from_pandas(val.sample(150))
98
  def preprocess_function(examples):
99
  inputs = [f"Summarize the text: {ex}" for ex in examples["source"]]
100
  targets = [f"Summary: {ex}" for ex in examples["target"]]