deve

app.py

@@ -38,10 +38,10 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                 preds = preds[0]
             # Replace -100s used for padding as we can't decode them
             preds = np.where(preds != -100, preds, tokenizer.pad_token_id)
-            preds = np.array(preds)
+            #preds = np.array(preds)
             decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
             labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
-            labels = np.array(labels)
+            #labels = np.array(labels)
             decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
       
             result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)
@@ -59,7 +59,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                  
         # Set training arguments
         training_args = TrainingArguments(
-            remove_unused_columns=False, 
+            remove_unused_columns=False,    
             torch_empty_cache_steps=100, 
             overwrite_output_dir=True, 
             output_dir='/data/results',
@@ -208,6 +208,18 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
             #print('DONE')
             #return 'RUN AGAIN TO LOAD REST OF DATA'
         dataset = load_dataset(dataset_name.strip())
+        for o, d in enumerate(dataset['validation']['text']):
+            if not isinstance(d, str):
+                print('hit')
+                print(type(d))
+                print(o)
+        for o, d in enumerate(dataset['validation']['target']):
+            if not isinstance(d, str):
+                print('hit')
+                print(type(d))
+                print(o)
+        return 'done'
+                
         #dataset['train'] = dataset['train'].select(range(8000))
         dataset['train'] = dataset['train'].select(range(4000))
         dataset['validation'] = dataset['validation'].select(range(200))