Spaces:
Running
on
A10G
Running
on
A10G
burtenshaw
commited on
Commit
Β·
8f5cc68
1
Parent(s):
d6ee53d
add push functionality and note about duplication
Browse files
app.py
CHANGED
@@ -68,9 +68,23 @@ def create_autotrain_params(
|
|
68 |
epochs: int,
|
69 |
batch_size: int,
|
70 |
learning_rate: float,
|
|
|
|
|
71 |
**kwargs,
|
72 |
):
|
73 |
"""Create AutoTrain parameter object based on task type"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
74 |
common_params = {
|
75 |
"model": base_model,
|
76 |
"project_name": project_name,
|
@@ -94,6 +108,7 @@ def create_autotrain_params(
|
|
94 |
"mixed_precision": "no",
|
95 |
"save_total_limit": 1,
|
96 |
"eval_strategy": "epoch",
|
|
|
97 |
}
|
98 |
|
99 |
if task == "text-classification":
|
@@ -114,12 +129,15 @@ def create_autotrain_params(
|
|
114 |
"llm-reward": "reward",
|
115 |
}
|
116 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
117 |
return LLMTrainingParams(
|
118 |
-
**
|
119 |
-
k: v
|
120 |
-
for k, v in common_params.items()
|
121 |
-
if k not in ["early_stopping_patience", "early_stopping_threshold"]
|
122 |
-
},
|
123 |
text_column=kwargs.get("text_column", "messages"),
|
124 |
block_size=kwargs.get("block_size", 2048),
|
125 |
peft=kwargs.get("use_peft", True),
|
@@ -245,6 +263,8 @@ def start_training_job(
|
|
245 |
batch_size: str = "8",
|
246 |
learning_rate: str = "2e-5",
|
247 |
backend: str = "local",
|
|
|
|
|
248 |
) -> str:
|
249 |
"""
|
250 |
Start a new AutoTrain training job.
|
@@ -260,6 +280,8 @@ def start_training_job(
|
|
260 |
batch_size: Training batch size (default: 16)
|
261 |
learning_rate: Learning rate for training (default: 2e-5)
|
262 |
backend: Training backend to use (default: local)
|
|
|
|
|
263 |
|
264 |
Returns:
|
265 |
Status message with run ID and details
|
@@ -269,6 +291,7 @@ def start_training_job(
|
|
269 |
epochs_int = int(epochs)
|
270 |
batch_size_int = int(batch_size)
|
271 |
learning_rate_float = float(learning_rate)
|
|
|
272 |
|
273 |
# Generate run ID
|
274 |
run_id = str(uuid.uuid4())
|
@@ -283,12 +306,16 @@ def start_training_job(
|
|
283 |
"status": "pending",
|
284 |
"created_at": datetime.utcnow().isoformat(),
|
285 |
"updated_at": datetime.utcnow().isoformat(),
|
|
|
|
|
286 |
"config": {
|
287 |
"task": task,
|
288 |
"epochs": epochs_int,
|
289 |
"batch_size": batch_size_int,
|
290 |
"learning_rate": learning_rate_float,
|
291 |
"backend": backend,
|
|
|
|
|
292 |
},
|
293 |
}
|
294 |
|
@@ -306,6 +333,8 @@ def start_training_job(
|
|
306 |
epochs=epochs_int,
|
307 |
batch_size=batch_size_int,
|
308 |
learning_rate=learning_rate_float,
|
|
|
|
|
309 |
)
|
310 |
|
311 |
# Start training in background
|
@@ -315,7 +344,8 @@ def start_training_job(
|
|
315 |
thread.daemon = True
|
316 |
thread.start()
|
317 |
|
318 |
-
|
|
|
319 |
|
320 |
Run ID: {run_id}
|
321 |
Project: {project_name}
|
@@ -327,7 +357,18 @@ Configuration:
|
|
327 |
β’ Epochs: {epochs}
|
328 |
β’ Batch Size: {batch_size}
|
329 |
β’ Learning Rate: {learning_rate}
|
330 |
-
β’ Backend: {backend}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
331 |
|
332 |
π Monitor progress:
|
333 |
β’ Gradio UI: http://localhost:7860
|
@@ -335,6 +376,8 @@ Configuration:
|
|
335 |
|
336 |
π‘ Use get_training_runs() to check status"""
|
337 |
|
|
|
|
|
338 |
except Exception as e:
|
339 |
return f"β Error submitting job: {str(e)}"
|
340 |
|
@@ -449,6 +492,18 @@ def get_run_details(run_id: str) -> str:
|
|
449 |
details_text += f"\nβ’ Learning Rate: {config.get('learning_rate')}"
|
450 |
details_text += f"\nβ’ Backend: {config.get('backend')}"
|
451 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
452 |
return details_text
|
453 |
|
454 |
except Exception as e:
|
@@ -656,6 +711,8 @@ def submit_training_job_ui(
|
|
656 |
batch_size,
|
657 |
learning_rate,
|
658 |
backend,
|
|
|
|
|
659 |
):
|
660 |
"""Submit training job from web UI"""
|
661 |
if not all([task, project_name, base_model, dataset_path]):
|
@@ -670,6 +727,8 @@ def submit_training_job_ui(
|
|
670 |
batch_size=str(batch_size),
|
671 |
learning_rate=str(learning_rate),
|
672 |
backend=backend,
|
|
|
|
|
673 |
)
|
674 |
|
675 |
return result, fetch_runs_for_ui()
|
@@ -685,14 +744,42 @@ with gr.Blocks(
|
|
685 |
}
|
686 |
""",
|
687 |
) as app:
|
688 |
-
gr.Markdown("""
|
689 |
# π AutoTrain Gradio MCP Server
|
690 |
|
691 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
692 |
|
693 |
-
β’ **Web Interface**: Manage training jobs through this UI
|
694 |
-
β’ **MCP Server**: AI assistants can use tools at `http://localhost:7860/gradio_api/mcp/sse`
|
695 |
-
β’ **Direct Integration**: No FastAPI needed - everything runs in Gradio
|
696 |
""")
|
697 |
|
698 |
with gr.Tabs():
|
@@ -716,6 +803,11 @@ with gr.Blocks(
|
|
716 |
with gr.Tab("π Start Training"):
|
717 |
gr.Markdown("## Submit New Training Job")
|
718 |
|
|
|
|
|
|
|
|
|
|
|
719 |
with gr.Row():
|
720 |
with gr.Column():
|
721 |
task_dropdown = gr.Dropdown(
|
@@ -750,6 +842,13 @@ with gr.Blocks(
|
|
750 |
value="local",
|
751 |
)
|
752 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
753 |
submit_btn = gr.Button("π Start Training", variant="primary", size="lg")
|
754 |
submit_output = gr.Textbox(label="Status", interactive=False, lines=10)
|
755 |
|
@@ -765,13 +864,27 @@ with gr.Blocks(
|
|
765 |
|
766 |
### Available MCP Tools:
|
767 |
|
768 |
-
- `start_training_job` - Submit new training jobs
|
769 |
- `get_training_runs` - List all runs with status
|
770 |
- `get_run_details` - Get detailed run information
|
771 |
-
- `delete_training_run` - Delete training runs
|
772 |
- `get_task_recommendations` - Get training recommendations
|
773 |
- `get_system_status` - Check system status
|
774 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
775 |
### Claude Desktop Configuration:
|
776 |
|
777 |
```json
|
@@ -788,6 +901,7 @@ with gr.Blocks(
|
|
788 |
|
789 |
Total Runs: {len(load_runs())}
|
790 |
W&B Project: {WANDB_PROJECT}
|
|
|
791 |
""")
|
792 |
|
793 |
# MCP Tools Tab
|
@@ -825,6 +939,8 @@ with gr.Blocks(
|
|
825 |
gr.Textbox(label="batch_size", value="8"),
|
826 |
gr.Textbox(label="learning_rate", value="2e-5"),
|
827 |
gr.Textbox(label="backend", value="local"),
|
|
|
|
|
828 |
],
|
829 |
outputs=gr.Textbox(label="Training Job Result"),
|
830 |
title="start_training_job",
|
@@ -875,6 +991,8 @@ with gr.Blocks(
|
|
875 |
batch_size,
|
876 |
learning_rate,
|
877 |
backend,
|
|
|
|
|
878 |
],
|
879 |
outputs=[submit_output, runs_table],
|
880 |
)
|
|
|
68 |
epochs: int,
|
69 |
batch_size: int,
|
70 |
learning_rate: float,
|
71 |
+
push_to_hub: bool,
|
72 |
+
hub_repo_id: str = "",
|
73 |
**kwargs,
|
74 |
):
|
75 |
"""Create AutoTrain parameter object based on task type"""
|
76 |
+
# Hub configuration
|
77 |
+
hub_config = {}
|
78 |
+
if push_to_hub:
|
79 |
+
hub_config = {
|
80 |
+
"push_to_hub": True,
|
81 |
+
"username": os.environ.get("HF_USERNAME", ""),
|
82 |
+
"token": os.environ.get("HF_TOKEN", ""),
|
83 |
+
}
|
84 |
+
# If custom repo_id is provided, use it; otherwise use project_name
|
85 |
+
if hub_repo_id:
|
86 |
+
hub_config["repo_id"] = hub_repo_id
|
87 |
+
|
88 |
common_params = {
|
89 |
"model": base_model,
|
90 |
"project_name": project_name,
|
|
|
108 |
"mixed_precision": "no",
|
109 |
"save_total_limit": 1,
|
110 |
"eval_strategy": "epoch",
|
111 |
+
**hub_config, # Add hub configuration
|
112 |
}
|
113 |
|
114 |
if task == "text-classification":
|
|
|
129 |
"llm-reward": "reward",
|
130 |
}
|
131 |
|
132 |
+
# For LLM tasks, exclude some parameters that don't apply
|
133 |
+
llm_params = {
|
134 |
+
k: v
|
135 |
+
for k, v in common_params.items()
|
136 |
+
if k not in ["early_stopping_patience", "early_stopping_threshold"]
|
137 |
+
}
|
138 |
+
|
139 |
return LLMTrainingParams(
|
140 |
+
**llm_params,
|
|
|
|
|
|
|
|
|
141 |
text_column=kwargs.get("text_column", "messages"),
|
142 |
block_size=kwargs.get("block_size", 2048),
|
143 |
peft=kwargs.get("use_peft", True),
|
|
|
263 |
batch_size: str = "8",
|
264 |
learning_rate: str = "2e-5",
|
265 |
backend: str = "local",
|
266 |
+
push_to_hub: str = "false",
|
267 |
+
hub_repo_id: str = "",
|
268 |
) -> str:
|
269 |
"""
|
270 |
Start a new AutoTrain training job.
|
|
|
280 |
batch_size: Training batch size (default: 16)
|
281 |
learning_rate: Learning rate for training (default: 2e-5)
|
282 |
backend: Training backend to use (default: local)
|
283 |
+
push_to_hub: Whether to push final model to Hub (true/false)
|
284 |
+
hub_repo_id: Custom repository ID for Hub (optional)
|
285 |
|
286 |
Returns:
|
287 |
Status message with run ID and details
|
|
|
291 |
epochs_int = int(epochs)
|
292 |
batch_size_int = int(batch_size)
|
293 |
learning_rate_float = float(learning_rate)
|
294 |
+
push_to_hub_bool = push_to_hub.lower() == "true"
|
295 |
|
296 |
# Generate run ID
|
297 |
run_id = str(uuid.uuid4())
|
|
|
306 |
"status": "pending",
|
307 |
"created_at": datetime.utcnow().isoformat(),
|
308 |
"updated_at": datetime.utcnow().isoformat(),
|
309 |
+
"push_to_hub": push_to_hub_bool,
|
310 |
+
"hub_repo_id": hub_repo_id,
|
311 |
"config": {
|
312 |
"task": task,
|
313 |
"epochs": epochs_int,
|
314 |
"batch_size": batch_size_int,
|
315 |
"learning_rate": learning_rate_float,
|
316 |
"backend": backend,
|
317 |
+
"push_to_hub": push_to_hub_bool,
|
318 |
+
"hub_repo_id": hub_repo_id,
|
319 |
},
|
320 |
}
|
321 |
|
|
|
333 |
epochs=epochs_int,
|
334 |
batch_size=batch_size_int,
|
335 |
learning_rate=learning_rate_float,
|
336 |
+
push_to_hub=push_to_hub_bool,
|
337 |
+
hub_repo_id=hub_repo_id,
|
338 |
)
|
339 |
|
340 |
# Start training in background
|
|
|
344 |
thread.daemon = True
|
345 |
thread.start()
|
346 |
|
347 |
+
# Build result message
|
348 |
+
result_msg = f"""β
Training job submitted successfully!
|
349 |
|
350 |
Run ID: {run_id}
|
351 |
Project: {project_name}
|
|
|
357 |
β’ Epochs: {epochs}
|
358 |
β’ Batch Size: {batch_size}
|
359 |
β’ Learning Rate: {learning_rate}
|
360 |
+
β’ Backend: {backend}"""
|
361 |
+
|
362 |
+
if push_to_hub_bool:
|
363 |
+
final_repo = hub_repo_id if hub_repo_id else project_name
|
364 |
+
result_msg += f"""
|
365 |
+
β’ Push to Hub: β
Enabled
|
366 |
+
β’ Repository: {final_repo}
|
367 |
+
β’ Requires: HF_USERNAME and HF_TOKEN environment variables"""
|
368 |
+
else:
|
369 |
+
result_msg += "\nβ’ Push to Hub: β Disabled"
|
370 |
+
|
371 |
+
result_msg += """
|
372 |
|
373 |
π Monitor progress:
|
374 |
β’ Gradio UI: http://localhost:7860
|
|
|
376 |
|
377 |
π‘ Use get_training_runs() to check status"""
|
378 |
|
379 |
+
return result_msg
|
380 |
+
|
381 |
except Exception as e:
|
382 |
return f"β Error submitting job: {str(e)}"
|
383 |
|
|
|
492 |
details_text += f"\nβ’ Learning Rate: {config.get('learning_rate')}"
|
493 |
details_text += f"\nβ’ Backend: {config.get('backend')}"
|
494 |
|
495 |
+
# Hub configuration
|
496 |
+
if config.get("push_to_hub"):
|
497 |
+
details_text += "\nβ’ Push to Hub: β
Enabled"
|
498 |
+
if config.get("hub_repo_id"):
|
499 |
+
details_text += f"\nβ’ Hub Repository: {config.get('hub_repo_id')}"
|
500 |
+
else:
|
501 |
+
details_text += (
|
502 |
+
f"\nβ’ Hub Repository: {run['project_name']} (default)"
|
503 |
+
)
|
504 |
+
else:
|
505 |
+
details_text += "\nβ’ Push to Hub: β Disabled"
|
506 |
+
|
507 |
return details_text
|
508 |
|
509 |
except Exception as e:
|
|
|
711 |
batch_size,
|
712 |
learning_rate,
|
713 |
backend,
|
714 |
+
push_to_hub,
|
715 |
+
hub_repo_id,
|
716 |
):
|
717 |
"""Submit training job from web UI"""
|
718 |
if not all([task, project_name, base_model, dataset_path]):
|
|
|
727 |
batch_size=str(batch_size),
|
728 |
learning_rate=str(learning_rate),
|
729 |
backend=backend,
|
730 |
+
push_to_hub=str(push_to_hub).lower(),
|
731 |
+
hub_repo_id=hub_repo_id,
|
732 |
)
|
733 |
|
734 |
return result, fetch_runs_for_ui()
|
|
|
744 |
}
|
745 |
""",
|
746 |
) as app:
|
747 |
+
gr.Markdown(f"""
|
748 |
# π AutoTrain Gradio MCP Server
|
749 |
|
750 |
+
Get your AI models to train your AI models!
|
751 |
+
|
752 |
+
This space is an MCP server that you can use in Claude Desktop, Cursor, VSCode, etc to train your AI models.
|
753 |
+
|
754 |
+
:warning: To train models you with need to duplicate this space!
|
755 |
+
**MCP Server**: AI assistants can use tools at http://SPACE_URL/gradio_api/mcp/sse
|
756 |
+
|
757 |
+
Connect to it like this:
|
758 |
+
|
759 |
+
```json
|
760 |
+
{"mcpServers": {"autotrain": {"url": "http://SPACE_URL/gradio_api/mcp/sse",
|
761 |
+
"headers": {"Authorization": "Bearer <YOUR-HUGGING-FACE-TOKEN>"
|
762 |
+
}
|
763 |
+
}
|
764 |
+
}
|
765 |
+
}
|
766 |
+
```
|
767 |
+
|
768 |
+
Or like this for Claude Desktop:
|
769 |
+
|
770 |
+
```json
|
771 |
+
{"mcpServers": {"hf-mcp-server": {"command": "npx",
|
772 |
+
"args": [
|
773 |
+
"mcp-remote",
|
774 |
+
"http://SPACE_URL/gradio_api/mcp/sse",
|
775 |
+
"--header",
|
776 |
+
"Authorization: Bearer <YOUR-HUGGING-FACE-TOKEN>"
|
777 |
+
]
|
778 |
+
}
|
779 |
+
}
|
780 |
+
}
|
781 |
+
```
|
782 |
|
|
|
|
|
|
|
783 |
""")
|
784 |
|
785 |
with gr.Tabs():
|
|
|
803 |
with gr.Tab("π Start Training"):
|
804 |
gr.Markdown("## Submit New Training Job")
|
805 |
|
806 |
+
gr.Markdown("""
|
807 |
+
π‘ **Hub Integration**: Enable "Push to Hub" to automatically upload your trained model to Hugging Face Hub.
|
808 |
+
Requires `HF_USERNAME` and `HF_TOKEN` environment variables.
|
809 |
+
""")
|
810 |
+
|
811 |
with gr.Row():
|
812 |
with gr.Column():
|
813 |
task_dropdown = gr.Dropdown(
|
|
|
842 |
value="local",
|
843 |
)
|
844 |
|
845 |
+
with gr.Row():
|
846 |
+
with gr.Column():
|
847 |
+
push_to_hub = gr.Checkbox(label="Push to Hub", value=False)
|
848 |
+
hub_repo_id = gr.Textbox(
|
849 |
+
label="Hub Repository ID", placeholder="your-repo-id"
|
850 |
+
)
|
851 |
+
|
852 |
submit_btn = gr.Button("π Start Training", variant="primary", size="lg")
|
853 |
submit_output = gr.Textbox(label="Status", interactive=False, lines=10)
|
854 |
|
|
|
864 |
|
865 |
### Available MCP Tools:
|
866 |
|
867 |
+
- `start_training_job` - Submit new training jobs (includes Hub push)
|
868 |
- `get_training_runs` - List all runs with status
|
869 |
- `get_run_details` - Get detailed run information
|
|
|
870 |
- `get_task_recommendations` - Get training recommendations
|
871 |
- `get_system_status` - Check system status
|
872 |
|
873 |
+
### π€ Hugging Face Hub Integration:
|
874 |
+
|
875 |
+
To push models to the Hub, set these environment variables:
|
876 |
+
|
877 |
+
```bash
|
878 |
+
export HF_USERNAME="your-hf-username"
|
879 |
+
export HF_TOKEN="your-hf-write-token"
|
880 |
+
```
|
881 |
+
|
882 |
+
Get your token from: https://huggingface.co/settings/tokens
|
883 |
+
|
884 |
+
**Usage Examples:**
|
885 |
+
- `push_to_hub="true"` - Push to Hub using project name as repo
|
886 |
+
- `hub_repo_id="my-org/my-model"` - Push to custom repository
|
887 |
+
|
888 |
### Claude Desktop Configuration:
|
889 |
|
890 |
```json
|
|
|
901 |
|
902 |
Total Runs: {len(load_runs())}
|
903 |
W&B Project: {WANDB_PROJECT}
|
904 |
+
Hub Auth: {"β
Configured" if os.environ.get("HF_TOKEN") else "β Missing HF_TOKEN"}
|
905 |
""")
|
906 |
|
907 |
# MCP Tools Tab
|
|
|
939 |
gr.Textbox(label="batch_size", value="8"),
|
940 |
gr.Textbox(label="learning_rate", value="2e-5"),
|
941 |
gr.Textbox(label="backend", value="local"),
|
942 |
+
gr.Textbox(label="push_to_hub", value="false"),
|
943 |
+
gr.Textbox(label="hub_repo_id", placeholder="your-repo-id"),
|
944 |
],
|
945 |
outputs=gr.Textbox(label="Training Job Result"),
|
946 |
title="start_training_job",
|
|
|
991 |
batch_size,
|
992 |
learning_rate,
|
993 |
backend,
|
994 |
+
push_to_hub,
|
995 |
+
hub_repo_id,
|
996 |
],
|
997 |
outputs=[submit_output, runs_table],
|
998 |
)
|