ふぁｄ

proxy_server.py

@@ -63,6 +63,9 @@ from litellm.caching import DualCache
 from litellm.proxy.health_check import perform_health_check
 from litellm._logging import verbose_router_logger, verbose_proxy_logger
 
+from utils import getenv, set_env_variables
+import litellm as llm
+
 litellm.suppress_debug_info = True
 from fastapi import (
     FastAPI,
@@ -1414,11 +1417,68 @@ async def completion(
         raise HTTPException(status_code=status, detail=error_msg)
 
 
+def completion_request(user_api_key):
+    resp = requests.post(
+        f"{FASTREPL_PROXY_URL_BASE}/chat/completions",
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {user_api_key}",
+        },
+        json={
+            "model": "gpt-3.5-turbo",
+            "messages": [
+                            { 
+                                "content": "what is YC?",
+                                "role": "user"
+                            }
+                        ]
+
+        }
+    )
+    return resp.json()
+
+
+if __name__ == "__main__":
+    key = new_user(100)["api_key"]
+    print("new key", key)
+
+    for _ in range(20):
+        print(cost_current(key))
+    
+    response = completion_request(key)
+    print(response)
+
+
+# for completion
+@routers.post("/v1/chat/completions", dependencies=[Depends(user_api_key_auth)])
+async def completion(request: Request):
+    key = request.headers.get("Authorization").replace("Bearer ", "")  # type: ignore
+    data = await request.json()
+    print(f"received request data: {data}")
+    data["user_key"] = key
+    data["budget_manager"] = budget_manager
+    data["master_key"] = master_key
+    set_env_variables(data)
+    # handle how users send streaming
+    if 'stream' in data:
+        if type(data['stream']) == str: # if users send stream as str convert to bool
+            # convert to bool
+            if data['stream'].lower() == "true":
+                data['stream'] = True # convert to boolean
+    
+    response = llm.completion(**data)
+    if 'stream' in data and data['stream'] == True: # use generate_responses to stream responses
+            return StreamingResponse(data_generator(response), media_type='text/event-stream')
+    return response
+
+
 @routers.post(
-    "/v1/chat/completions",
+    "/v1/chat/completionsssssssssss",
     dependencies=[Depends(user_api_key_auth)],
     tags=["chat/completions"],
 )
+def test(aaaa,bbbb):
+    print(aaa)
 
 
 @router.post(

utils.py

@@ -0,0 +1,21 @@
+import os
+import functools
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+@functools.lru_cache(maxsize=None)
+def getenv(key, default=0):
+    return type(default)(os.getenv(key, default))
+
+
+def set_env_variables(data):
+    try:
+        if "env_variables" in data:
+            env_variables = data["env_variables"]
+            for key in env_variables:
+                os.environ[key] = env_variables[key]
+            data.pop("env_variables")
+    except:
+        pass