Update handler.py
Browse files- handler.py +6 -0
handler.py
CHANGED
|
@@ -2,6 +2,12 @@ from typing import Dict, List, Any
|
|
| 2 |
from llama_cpp import Llama
|
| 3 |
import torch
|
| 4 |
from loguru import logger
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
|
| 6 |
MAX_INPUT_TOKEN_LENGTH = 4000
|
| 7 |
MAX_MAX_NEW_TOKENS = 2048
|
|
|
|
| 2 |
from llama_cpp import Llama
|
| 3 |
import torch
|
| 4 |
from loguru import logger
|
| 5 |
+
import time
|
| 6 |
+
|
| 7 |
+
import psutil
|
| 8 |
+
|
| 9 |
+
_ = psutil.cpu_count(logical=True)
|
| 10 |
+
cpu_count: int = int(_) if _ else 1
|
| 11 |
|
| 12 |
MAX_INPUT_TOKEN_LENGTH = 4000
|
| 13 |
MAX_MAX_NEW_TOKENS = 2048
|