init

app.py

@@ -40,24 +40,23 @@ def fine_tune_model(model_name, dataset_name, hub_id, api_key, num_epochs, batch
     
         # Tokenize the dataset
         def tokenize_function(examples):
-            max_length = 16
+            max_length = 128
             # Assuming 'text' is the input and 'target' is the expected output
             model_inputs = tokenizer(
                 examples['text'], 
-                max_length=None,  # Set to None for dynamic padding
+                max_length=max_length,  # Set to None for dynamic padding
                 padding=False,     # Disable padding here, we will handle it later
                 truncation=True,
-                return_tensors="pt"  # Return PyTorch tensors
             )
         
             # Setup the decoder input IDs (shifted right)
             with tokenizer.as_target_tokenizer():
                 labels = tokenizer(
                     examples['target'], 
-                    max_length=None,  # Set to None for dynamic padding
+                    max_length=max_length,  # Set to None for dynamic padding
                     padding=False,     # Disable padding here, we will handle it later
                     truncation=True,
-                    return_tensors="pt"                    
+                
                 )
         
             # Add labels to the model inputs