Spaces:
Runtime error
Runtime error
Commit
·
efd3dc5
1
Parent(s):
266a3d3
Fixed macos issue
Browse files- main/main.py +17 -7
main/main.py
CHANGED
@@ -1,6 +1,8 @@
|
|
1 |
"""
|
2 |
LLM Inference Server main application using LitServe framework.
|
3 |
"""
|
|
|
|
|
4 |
import litserve as ls
|
5 |
import logging
|
6 |
import os
|
@@ -52,13 +54,21 @@ def create_app():
|
|
52 |
# Initialize router with API instance
|
53 |
init_router(api, config)
|
54 |
|
55 |
-
#
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
62 |
|
63 |
# Launch inference workers (assuming single uvicorn worker for now)
|
64 |
_MANAGER, _WORKER_PROCESSES = server.launch_inference_worker(num_uvicorn_servers=1)
|
|
|
1 |
"""
|
2 |
LLM Inference Server main application using LitServe framework.
|
3 |
"""
|
4 |
+
from sys import platform
|
5 |
+
|
6 |
import litserve as ls
|
7 |
import logging
|
8 |
import os
|
|
|
54 |
# Initialize router with API instance
|
55 |
init_router(api, config)
|
56 |
|
57 |
+
if platform == "darwin": # Darwin is macOS
|
58 |
+
server = ls.LitServer(
|
59 |
+
api,
|
60 |
+
timeout=server_config.get('timeout', 60),
|
61 |
+
max_batch_size=server_config.get('max_batch_size', 1),
|
62 |
+
track_requests=True,
|
63 |
+
accelerator="cpu" # Force CPU on Mac
|
64 |
+
)
|
65 |
+
else:
|
66 |
+
server = ls.LitServer(
|
67 |
+
api,
|
68 |
+
timeout=server_config.get('timeout', 60),
|
69 |
+
max_batch_size=server_config.get('max_batch_size', 1),
|
70 |
+
track_requests=True
|
71 |
+
)
|
72 |
|
73 |
# Launch inference workers (assuming single uvicorn worker for now)
|
74 |
_MANAGER, _WORKER_PROCESSES = server.launch_inference_worker(num_uvicorn_servers=1)
|