dev

app.py

@@ -8,6 +8,8 @@ import os
 from huggingface_hub import login
 from peft import get_peft_model, LoraConfig
 
+os.environ('HF_HOME', '/data/.huggingface')
+
 @spaces.GPU(duration=120)
 def fine_tune_model(model_name, dataset_name, hub_id, api_key, num_epochs, batch_size, lr, grad):
     try:        
@@ -28,8 +30,8 @@ def fine_tune_model(model_name, dataset_name, hub_id, api_key, num_epochs, batch
     
         max_length = 128
         try:
-            tokenized_train_dataset = load_from_disk(f'data/{hub_id.strip()}_train_dataset')
-            tokenized_test_dataset = load_from_disk(f'data/{hub_id.strip()}_test_dataset')
+            tokenized_train_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')
+            tokenized_test_dataset = load_from_disk(f'/data/{hub_id.strip()}_test_dataset')
             tokenized_datasets = concatenate_datasets([tokenized_train_dataset, tokenized_test_dataset])
         except:
             # Tokenize the dataset
@@ -58,8 +60,8 @@ def fine_tune_model(model_name, dataset_name, hub_id, api_key, num_epochs, batch
         
             tokenized_datasets = dataset.map(tokenize_function, batched=True, batch_size=32)
             
-            tokenized_datasets['train'].save_to_disk(f'data/{hub_id.strip()}_train_dataset')
-            tokenized_datasets['test'].save_to_disk(f'data/{hub_id.strip()}_test_dataset')
+            tokenized_datasets['train'].save_to_disk(f'/data/{hub_id.strip()}_train_dataset')
+            tokenized_datasets['test'].save_to_disk(f'/data/{hub_id.strip()}_test_dataset')
         
     
         # Set training arguments