AurelioAguirre commited on
Commit
efd3dc5
·
1 Parent(s): 266a3d3

Fixed macos issue

Browse files
Files changed (1) hide show
  1. main/main.py +17 -7
main/main.py CHANGED
@@ -1,6 +1,8 @@
1
  """
2
  LLM Inference Server main application using LitServe framework.
3
  """
 
 
4
  import litserve as ls
5
  import logging
6
  import os
@@ -52,13 +54,21 @@ def create_app():
52
  # Initialize router with API instance
53
  init_router(api, config)
54
 
55
- # Create LitServer instance
56
- server = ls.LitServer(
57
- api,
58
- timeout=server_config.get('timeout', 60),
59
- max_batch_size=server_config.get('max_batch_size', 1),
60
- track_requests=True
61
- )
 
 
 
 
 
 
 
 
62
 
63
  # Launch inference workers (assuming single uvicorn worker for now)
64
  _MANAGER, _WORKER_PROCESSES = server.launch_inference_worker(num_uvicorn_servers=1)
 
1
  """
2
  LLM Inference Server main application using LitServe framework.
3
  """
4
+ from sys import platform
5
+
6
  import litserve as ls
7
  import logging
8
  import os
 
54
  # Initialize router with API instance
55
  init_router(api, config)
56
 
57
+ if platform == "darwin": # Darwin is macOS
58
+ server = ls.LitServer(
59
+ api,
60
+ timeout=server_config.get('timeout', 60),
61
+ max_batch_size=server_config.get('max_batch_size', 1),
62
+ track_requests=True,
63
+ accelerator="cpu" # Force CPU on Mac
64
+ )
65
+ else:
66
+ server = ls.LitServer(
67
+ api,
68
+ timeout=server_config.get('timeout', 60),
69
+ max_batch_size=server_config.get('max_batch_size', 1),
70
+ track_requests=True
71
+ )
72
 
73
  # Launch inference workers (assuming single uvicorn worker for now)
74
  _MANAGER, _WORKER_PROCESSES = server.launch_inference_worker(num_uvicorn_servers=1)