changing to routing proxy

aiproxy/__init__.py

@@ -0,0 +1,14 @@
+from .proxy import (
+    RequestFilterBase,
+    ResponseFilterBase,
+    RequestFilterException,
+    ResponseFilterException
+)
+
+from .accesslog import (
+    AccessLogBase,
+    AccessLog,
+    AccessLogWorker
+)
+
+from .chatgpt import ChatGPTProxy

aiproxy/__main__.py

@@ -0,0 +1,49 @@
+import argparse
+from contextlib import asynccontextmanager
+import logging
+import os
+from fastapi import FastAPI
+from aiproxy.chatgpt import ChatGPTProxy
+from aiproxy.accesslog import AccessLogWorker
+import threading
+import uvicorn
+
+# Get API Key from env
+env_openai_api_key = "test"
+
+# Get arguments
+parser = argparse.ArgumentParser(description="UnaProxy usage")
+parser.add_argument("--host", type=str, default="127.0.0.1", required=False, help="hostname or ipaddress")
+parser.add_argument("--port", type=int, default="7860", required=False, help="port number")
+parser.add_argument("--base_url", type=str, default="http://localhost:8000/v1/", required=False, help="port number")
+parser.add_argument("--openai_api_key", type=str, default=env_openai_api_key, required=False, help="OpenAI API Key")
+args = parser.parse_args()
+
+# Setup logger
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+log_format = logging.Formatter("%(asctime)s %(levelname)8s %(message)s")
+streamHandler = logging.StreamHandler()
+streamHandler.setFormatter(log_format)
+logger.addHandler(streamHandler)
+
+# Setup access log worker
+worker = AccessLogWorker()
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Start access log worker
+    threading.Thread(target=worker.run, daemon=True).start()
+    yield
+    # Stop access log worker
+    worker.queue_client.put(None)
+
+# Setup ChatGPTProxy
+proxy = ChatGPTProxy(base_url=args.base_url, api_key=args.openai_api_key, access_logger_queue=worker.queue_client)
+
+# Setup server application
+app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
+proxy.add_route(app, "/v1/chat/completions")
+#proxy.add_completion_route(app, "/v1/completions")
+
+uvicorn.run(app, host=args.host, port=args.port)

aiproxy/accesslog.py

@@ -0,0 +1,262 @@
+from abc import abstractmethod
+from datetime import datetime
+import json
+import logging
+from time import sleep
+import traceback
+from sqlalchemy import Column, Integer, String, Float, DateTime, create_engine
+from sqlalchemy.orm import sessionmaker, declarative_base, declared_attr, Session
+from .queueclient import DefaultQueueClient, QueueItemBase, QueueClientBase
+
+
+logger = logging.getLogger(__name__)
+
+
+class _AccessLogBase:
+    @declared_attr
+    def __tablename__(cls):
+        return cls.__name__.lower()
+
+    @declared_attr
+    def id(cls):
+        return Column(Integer, primary_key=True)
+
+    @declared_attr
+    def request_id(cls):
+        return Column(String)
+
+    @declared_attr
+    def created_at(cls):
+        return Column(DateTime)
+
+    @declared_attr
+    def direction(cls):
+        return Column(String)
+
+    @declared_attr
+    def status_code(cls):
+        return Column(Integer)
+
+    @declared_attr
+    def content(cls):
+        return Column(String)
+
+    @declared_attr
+    def function_call(cls):
+        return Column(String)
+
+    @declared_attr
+    def tool_calls(cls):
+        return Column(String)
+
+    @declared_attr
+    def raw_body(cls):
+        return Column(String)
+
+    @declared_attr
+    def raw_headers(cls):
+        return Column(String)
+
+    @declared_attr
+    def model(cls):
+        return Column(String)
+
+    @declared_attr
+    def prompt_tokens(cls):
+        return Column(Integer)
+
+    @declared_attr
+    def completion_tokens(cls):
+        return Column(Integer)
+
+    @declared_attr
+    def request_time(cls):
+        return Column(Float)
+
+    @declared_attr
+    def request_time_api(cls):
+        return Column(Float)
+
+
+# Classes for access log queue item
+class RequestItemBase(QueueItemBase):
+    def __init__(self, request_id: str, request_json: dict, request_headers: dict) -> None:
+        self.request_id = request_id
+        self.request_json = request_json
+        self.request_headers = request_headers
+
+    @abstractmethod
+    def to_accesslog(self, accesslog_cls: _AccessLogBase) -> _AccessLogBase:
+        ...
+
+
+class ResponseItemBase(QueueItemBase):
+    def __init__(self, request_id: str, response_json: dict, response_headers: dict = None, duration: float = 0, duration_api: float = 0, status_code: int = 0) -> None:
+        self.request_id = request_id
+        self.response_json = response_json
+        self.response_headers = response_headers
+        self.duration = duration
+        self.duration_api = duration_api
+        self.status_code = status_code
+
+    @abstractmethod
+    def to_accesslog(self, accesslog_cls: _AccessLogBase) -> _AccessLogBase:
+        ...
+
+
+class StreamChunkItemBase(QueueItemBase):
+    def __init__(self, request_id: str, chunk_json: dict = None, response_headers: dict = None, duration: float = 0, duration_api: float = 0, request_json: dict = None, status_code: int = 0) -> None:
+        self.request_id = request_id
+        self.chunk_json = chunk_json
+        self.response_headers = response_headers
+        self.duration = duration
+        self.duration_api = duration_api
+        self.request_json = request_json
+        self.status_code = status_code
+
+    @abstractmethod
+    def to_accesslog(self, chunks: list, accesslog_cls: _AccessLogBase) -> _AccessLogBase:
+        ...
+
+
+class ErrorItemBase(QueueItemBase):
+    def __init__(self, request_id: str, exception: Exception, traceback_info: str, response_json: dict = None, response_headers: dict = None, status_code: int = 0) -> None:
+        self.request_id = request_id
+        self.exception = exception
+        self.traceback_info = traceback_info
+        self.response_json = response_json
+        self.response_headers = response_headers
+        self.status_code = status_code
+
+    def to_accesslog(self, accesslog_cls: _AccessLogBase) -> _AccessLogBase:
+        if isinstance(self.response_json, dict):
+            try:
+                raw_body = json.dumps(self.response_json, ensure_ascii=False)
+            except Exception:
+                raw_body = str(self.response_json)
+        else:
+            raw_body = str(self.response_json)
+
+        return accesslog_cls(
+            request_id=self.request_id,
+            created_at=datetime.utcnow(),
+            direction="error",
+            content=f"{self.exception}\n{self.traceback_info}",
+            raw_body=raw_body,
+            raw_headers=json.dumps(self.response_headers, ensure_ascii=False) if self.response_headers else None,
+            model="error_handler",
+            status_code=self.status_code
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            "type": self.__class__.__name__,
+            "request_id": self.request_id,
+            "exception": str(self.exception),
+            "traceback_info": self.traceback_info,
+            "response_json": self.response_json,
+            "response_headers": self.response_headers
+        }
+
+
+class WorkerShutdownItem(QueueItemBase):
+    ...
+
+
+AccessLogBase = declarative_base(cls=_AccessLogBase)
+
+
+class AccessLog(AccessLogBase): ...
+
+
+class AccessLogWorker:
+    def __init__(self, *, connection_str: str = "sqlite:///aiproxy.db", db_engine = None, accesslog_cls = AccessLog, queue_client: QueueClientBase = None):
+        if db_engine:
+            self.db_engine = db_engine
+        else:
+            self.db_engine = create_engine(connection_str)
+        self.accesslog_cls = accesslog_cls
+        self.accesslog_cls.metadata.create_all(bind=self.db_engine)
+        self.get_session = sessionmaker(autocommit=False, autoflush=False, bind=self.db_engine)
+        self.queue_client = queue_client or DefaultQueueClient()
+        self.chunk_buffer = {}
+
+    def insert_request(self, accesslog: _AccessLogBase, db: Session):
+        db.add(accesslog)
+        db.commit()
+
+    def insert_response(self, accesslog: _AccessLogBase, db: Session):
+        db.add(accesslog)
+        db.commit()
+
+    def use_db(self, item: QueueItemBase):
+        return not (isinstance(item, StreamChunkItemBase) and item.duration == 0)
+
+    def process_item(self, item: QueueItemBase, db: Session):
+        try:
+            # Request
+            if isinstance(item, RequestItemBase):
+                self.insert_request(item.to_accesslog(self.accesslog_cls), db)
+
+            # Non-stream response
+            elif isinstance(item, ResponseItemBase):
+                self.insert_response(item.to_accesslog(self.accesslog_cls), db)
+
+            # Stream response
+            elif isinstance(item, StreamChunkItemBase):
+                if not self.chunk_buffer.get(item.request_id):
+                    self.chunk_buffer[item.request_id] = []
+
+                if item.duration == 0:
+                    self.chunk_buffer[item.request_id].append(item)
+
+                else:
+                    # Last chunk data for specific request_id
+                    self.insert_response(item.to_accesslog(
+                        self.chunk_buffer[item.request_id], self.accesslog_cls
+                    ), db)
+                    # Remove chunks from buffer
+                    del self.chunk_buffer[item.request_id]
+
+            # Error response
+            elif isinstance(item, ErrorItemBase):
+                self.insert_response(item.to_accesslog(self.accesslog_cls), db)
+
+        except Exception as ex:
+            logger.error(f"Error at processing queue item: {ex}\n{traceback.format_exc()}")
+
+
+    def run(self):
+        while True:
+            sleep(self.queue_client.dequeue_interval)
+            db = None
+            try:
+                items = self.queue_client.get()
+            except Exception as ex:
+                logger.error(f"Error at getting items from queue client: {ex}\n{traceback.format_exc()}")
+                continue
+
+            for item in items:
+                try:
+                    if isinstance(item, WorkerShutdownItem) or item is None:
+                        return
+
+                    if db is None and self.use_db(item):
+                        # Get db session just once in the loop when the item that uses db found
+                        db = self.get_session()
+
+                    self.process_item(item, db)
+
+                except Exception as pex:
+                    logger.error(f"Error at processing loop: {pex}\n{traceback.format_exc()}")
+                    # Try to persist data in error log instead
+                    try:
+                        logger.error(f"data: {item.to_json()}")
+                    except:
+                        logger.error(f"data(to_json() failed): {str(item)}")
+
+            if db is not None:
+                try:
+                    db.close()
+                except Exception as dbex:
+                    logger.error(f"Error at closing db session: {dbex}\n{traceback.format_exc()}")

aiproxy/async_proxy.py

@@ -0,0 +1,70 @@
+from fastapi import FastAPI, Request
+import httpx
+from starlette.responses import StreamingResponse, JSONResponse
+from starlette.background import BackgroundTask
+import uvicorn
+import json
+
+app = FastAPI(debug=True)
+
+# Define the base URL of your backend server
+BACKEND_BASE_URL = "http://localhost:8000"
+TIMEOUT_KEEP_ALIVE = 5.0
+timeout_config = httpx.Timeout(5.0, read=60.0)
+
+
+async def hook(response: httpx.Response) -> None:
+    if response.is_error:
+        await response.aread()
+        response.raise_for_status()
+
+
+@app.get("/{path:path}")
+async def forward_get_request(path: str, request: Request):
+    async with httpx.AsyncClient() as client:
+        response = await client.get(f"{BACKEND_BASE_URL}/{path}", params=request.query_params)
+        content = response.aiter_bytes() if response.is_stream_consumed else response.content
+        return StreamingResponse(content, media_type=response.headers['Content-Type'])
+
+
+@app.post("/{path:path}")
+async def forward_post_request(path: str, request: Request):
+    # Retrieve the request body
+    body = await request.body()
+
+    # Prepare the headers, excluding those that can cause issues
+    headers = {k: v for k, v in request.headers.items() if k.lower() not in ["host", "content-length"]}
+
+    async with httpx.AsyncClient(event_hooks={'response': [hook]}, timeout=timeout_config) as client:
+        # Send the request and get the response as a stream
+        req = client.build_request("POST", f"{BACKEND_BASE_URL}/{path}", content=body, headers=headers)
+
+        try:
+            response = await client.send(req, stream=True)
+            response.raise_for_status()
+
+            if json.loads(body.decode('utf-8'))['stream']:
+                # Custom streaming function
+                async def stream_response(response):
+                    async for chunk in response.aiter_bytes():
+                        yield chunk
+                    await response.aclose()  # Ensure the response is closed after streaming
+
+                return StreamingResponse(stream_response(response),
+                                         status_code=response.status_code,
+                                         headers=headers)
+            else:  # For regular JSON responses
+                # For non-streaming responses, read the complete response body
+                content = await response.aread()
+                return JSONResponse(content=content, status_code=response.status_code)
+        except httpx.ResponseNotRead as exc:
+            print(f"HTTP Exception for {exc.request.url} - {exc}")
+
+
+if __name__ == "__main__":
+    uvicorn.run(app,
+                host='127.0.0.1',
+                port=7860,
+                log_level="debug",
+                timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
+

aiproxy/chatgpt.py

@@ -0,0 +1,531 @@
+from datetime import datetime
+import json
+import logging
+import os
+import time
+import traceback
+from typing import List, Union, AsyncGenerator
+from uuid import uuid4
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+from sse_starlette.sse import EventSourceResponse, AsyncContentStream
+from openai import AsyncClient, APIStatusError, APIResponseValidationError, APIError, OpenAIError
+from openai.types.chat import ChatCompletion
+import tiktoken
+from .proxy import ProxyBase, RequestFilterBase, ResponseFilterBase, RequestFilterException, ResponseFilterException
+from .accesslog import _AccessLogBase, RequestItemBase, ResponseItemBase, StreamChunkItemBase, ErrorItemBase
+from .queueclient import QueueClientBase
+
+logger = logging.getLogger(__name__)
+
+
+class ChatGPTRequestItem(RequestItemBase):
+    def to_accesslog(self, accesslog_cls: _AccessLogBase) -> _AccessLogBase:
+        request_headers_copy = self.request_headers.copy()
+        if auth := request_headers_copy.get("authorization"):
+            request_headers_copy["authorization"] = auth[:12] + "*****" + auth[-2:]
+
+        content = self.request_json["messages"][-1]["content"]
+        if isinstance(content, list):
+            for c in content:
+                if c["type"] == "text":
+                    content = c["text"]
+                    break
+            else:
+                content = json.dumps(content)
+
+        accesslog = accesslog_cls(
+            request_id=self.request_id,
+            created_at=datetime.utcnow(),
+            direction="request",
+            content=content,
+            raw_body=json.dumps(self.request_json, ensure_ascii=False),
+            raw_headers=json.dumps(request_headers_copy, ensure_ascii=False),
+            model=self.request_json.get("model")
+        )
+
+        return accesslog
+
+
+class ChatGPTResponseItem(ResponseItemBase):
+    def to_accesslog(self, accesslog_cls: _AccessLogBase) -> _AccessLogBase:
+        content = self.response_json["choices"][0]["message"].get("content")
+        function_call = self.response_json["choices"][0]["message"].get("function_call")
+        tool_calls = self.response_json["choices"][0]["message"].get("tool_calls")
+        response_headers = json.dumps(dict(self.response_headers.items()),
+                                      ensure_ascii=False) if self.response_headers is not None else None
+        model = self.response_json["model"]
+        prompt_tokens = self.response_json["usage"]["prompt_tokens"]
+        completion_tokens = self.response_json["usage"]["completion_tokens"]
+
+        return accesslog_cls(
+            request_id=self.request_id,
+            created_at=datetime.utcnow(),
+            direction="response",
+            status_code=self.status_code,
+            content=content,
+            function_call=json.dumps(function_call, ensure_ascii=False) if function_call is not None else None,
+            tool_calls=json.dumps(tool_calls, ensure_ascii=False) if tool_calls is not None else None,
+            raw_body=json.dumps(self.response_json, ensure_ascii=False),
+            raw_headers=response_headers,
+            model=model,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            request_time=self.duration,
+            request_time_api=self.duration_api
+        )
+
+
+token_encoder = tiktoken.get_encoding("cl100k_base")
+
+
+def count_token(content: str):
+    return len(token_encoder.encode(content))
+
+
+def count_request_token(request_json: dict):
+    tokens_per_message = 3
+    tokens_per_name = 1
+    token_count = 0
+
+    # messages
+    for m in request_json["messages"]:
+        token_count += tokens_per_message
+        for k, v in m.items():
+            if isinstance(v, list):
+                for c in v:
+                    if c.get("type") == "text":
+                        token_count += count_token(c["text"])
+            else:
+                token_count += count_token(v)
+            if k == "name":
+                token_count += tokens_per_name
+
+    # functions
+    if functions := request_json.get("functions"):
+        for f in functions:
+            token_count += count_token(json.dumps(f))
+
+    # function_call
+    if function_call := request_json.get("function_call"):
+        if isinstance(function_call, dict):
+            token_count += count_token(json.dumps(function_call))
+        else:
+            token_count += count_token(function_call)
+
+    # tools
+    if tools := request_json.get("tools"):
+        for t in tools:
+            token_count += count_token(json.dumps(t))
+
+    if tool_choice := request_json.get("tool_choice"):
+        token_count += count_token(json.dumps(tool_choice))
+
+    token_count += 3
+    return token_count
+
+
+class ChatGPTStreamResponseItem(StreamChunkItemBase):
+    def to_accesslog(self, chunks: list, accesslog_cls: _AccessLogBase) -> _AccessLogBase:
+        chunk_jsons = []
+        response_content = ""
+        function_call = None
+        tool_calls = None
+        prompt_tokens = 0
+        completion_tokens = 0
+
+        # Parse info from chunks
+        for chunk in chunks:
+            chunk_jsons.append(chunk.chunk_json)
+
+            if len(chunk.chunk_json["choices"]) == 0:
+                # Azure returns the first delta with empty choices
+                continue
+
+            delta = chunk.chunk_json["choices"][0]["delta"]
+
+            # Make tool_calls
+            if delta.get("tool_calls"):
+                if tool_calls is None:
+                    tool_calls = []
+                if delta["tool_calls"][0]["function"].get("name"):
+                    tool_calls.append({
+                        "type": "function",
+                        "function": {
+                            "name": delta["tool_calls"][0]["function"]["name"],
+                            "arguments": ""
+                        }
+                    })
+                elif delta["tool_calls"][0]["function"].get("arguments"):
+                    tool_calls[-1]["function"]["arguments"] += delta["tool_calls"][0]["function"].get("arguments") or ""
+
+            # Make function_call
+            elif delta.get("function_call"):
+                if function_call is None:
+                    function_call = {}
+                if delta["function_call"].get("name"):
+                    function_call["name"] = delta["function_call"]["name"]
+                    function_call["arguments"] = ""
+                elif delta["function_call"].get("arguments"):
+                    function_call["arguments"] += delta["function_call"]["arguments"]
+
+            # Text content
+            else:
+                response_content += delta.get("content") or ""
+
+        # Serialize
+        function_call_str = json.dumps(function_call, ensure_ascii=False) if function_call is not None else None
+        tool_calls_str = json.dumps(tool_calls, ensure_ascii=False) if tool_calls is not None else None
+        response_headers = json.dumps(dict(self.response_headers.items()),
+                                      ensure_ascii=False) if self.response_headers is not None else None
+
+        # Count tokens
+        prompt_tokens = count_request_token(self.request_json)
+
+        if tool_calls_str:
+            completion_tokens = count_token(tool_calls_str)
+        elif function_call_str:
+            completion_tokens = count_token(function_call_str)
+        else:
+            completion_tokens = count_token(response_content)
+
+        return accesslog_cls(
+            request_id=self.request_id,
+            created_at=datetime.utcnow(),
+            direction="response",
+            status_code=self.status_code,
+            content=response_content,
+            function_call=function_call_str,
+            tool_calls=tool_calls_str,
+            raw_body=json.dumps(chunk_jsons, ensure_ascii=False),
+            raw_headers=response_headers,
+            model=chunk_jsons[0]["model"],
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            request_time=self.duration,
+            request_time_api=self.duration_api
+        )
+
+
+class ChatGPTErrorItem(ErrorItemBase):
+    ...
+
+
+queue_item_types = [ChatGPTRequestItem, ChatGPTResponseItem, ChatGPTStreamResponseItem, ChatGPTErrorItem]
+
+
+# Reverse aiproxy application for ChatGPT
+class ChatGPTProxy(ProxyBase):
+    _empty_openai_api_key = "OPENAI_API_KEY_IS_NOT_SET"
+
+    def __init__(
+            self,
+            *,
+            base_url: str = None,
+            api_key: str = None,
+            async_client: AsyncClient = None,
+            max_retries: int = 0,
+            timeout: float = 60.0,
+            request_filters: List[RequestFilterBase] = None,
+            response_filters: List[ResponseFilterBase] = None,
+            request_item_class: type = ChatGPTRequestItem,
+            response_item_class: type = ChatGPTResponseItem,
+            stream_response_item_class: type = ChatGPTStreamResponseItem,
+            error_item_class: type = ChatGPTErrorItem,
+            access_logger_queue: QueueClientBase,
+    ):
+        super().__init__(
+            request_filters=request_filters,
+            response_filters=response_filters,
+            access_logger_queue=access_logger_queue
+        )
+
+        # Log items
+        self.request_item_class = request_item_class
+        self.response_item_class = response_item_class
+        self.stream_response_item_class = stream_response_item_class
+        self.error_item_class = error_item_class
+
+        # ChatGPT client config
+        self.base_url = base_url
+        self.api_key = api_key or os.getenv("OPENAI_API_KEY") or self._empty_openai_api_key
+        self.max_retries = max_retries
+        self.timeout = timeout
+        self.async_client = async_client
+
+    async def filter_request(self, request_id: str, request_json: dict, request_headers: dict) -> Union[
+        dict, JSONResponse, EventSourceResponse]:
+        for f in self.request_filters:
+            if json_resp := await f.filter(request_id, request_json, request_headers):
+                # Return response if filter returns string
+                resp_for_log = {
+                    "id": "-",
+                    "choices": [
+                        {"message": {"role": "assistant", "content": json_resp}, "finish_reason": "stop", "index": 0}],
+                    "created": 0,
+                    "model": "request_filter",
+                    "object": "chat.completion",
+                    "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
+                }
+                # Response log
+                self.access_logger_queue.put(self.response_item_class(
+                    request_id=request_id,
+                    response_json=resp_for_log,
+                    status_code=200
+                ))
+
+                if request_json.get("stream"):
+                    # Stream
+                    async def filter_response_stream(content: str):
+                        # First delta
+                        resp = {
+                            "id": "-",
+                            "choices": [
+                                {"delta": {"role": "assistant", "content": ""}, "finish_reason": None, "index": 0}],
+                            "created": 0,
+                            "model": "request_filter",
+                            "object": "chat.completion",
+                            "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
+                        }
+                        yield json.dumps(resp)
+                        # Last delta
+                        resp["choices"][0] = {"delta": {"content": content}, "finish_reason": "stop", "index": 0}
+                        yield json.dumps(resp)
+
+                    return self.return_response_with_headers(EventSourceResponse(
+                        filter_response_stream(json_resp)
+                    ), request_id)
+
+                else:
+                    # Non-stream
+                    return self.return_response_with_headers(JSONResponse(resp_for_log), request_id)
+
+        return request_json
+
+    def get_client(self):
+        return self.async_client or AsyncClient(
+            base_url=self.base_url,
+            api_key=self.api_key,
+            max_retries=self.max_retries,
+            timeout=self.timeout
+        )
+
+    async def filter_response(self, request_id: str, response: ChatCompletion) -> ChatCompletion:
+        response_json = response.model_dump()
+
+        for f in self.response_filters:
+            if json_resp := await f.filter(request_id, response_json):
+                return response.model_validate(json_resp)
+
+        return response.model_validate(response_json)
+
+    def return_response_with_headers(self, resp: JSONResponse, request_id: str):
+        self.add_response_headers(response=resp, request_id=request_id)
+        return resp
+
+    def add_route(self, app: FastAPI, base_url: str):
+        @app.post(base_url)
+        async def handle_request(request: Request):
+            request_id = str(uuid4())
+            async_client = None
+
+            try:
+                start_time = time.time()
+                request_json = await request.json()
+                request_headers = dict(request.headers.items())
+
+                # Log request
+                self.access_logger_queue.put(self.request_item_class(
+                    request_id=request_id,
+                    request_json=request_json,
+                    request_headers=request_headers
+                ))
+
+                # Filter request
+                request_json = await self.filter_request(request_id, request_json, request_headers)
+                if isinstance(request_json, JSONResponse) or isinstance(request_json, EventSourceResponse):
+                    return request_json
+
+                # Call API
+                async_client = self.get_client()
+                start_time_api = time.time()
+                if self.api_key != self._empty_openai_api_key:
+                    # Always use server api key if set to client
+                    raw_response = await async_client.chat.completions.with_raw_response.create(**request_json)
+                elif user_auth_header := request_headers.get("authorization"):  # Lower case from client.
+                    raw_response = await async_client.chat.completions.with_raw_response.create(
+                        **request_json, extra_headers={"Authorization": user_auth_header}  # Pascal to server
+                    )
+                else:
+                    # Call API anyway ;)
+                    raw_response = await async_client.chat.completions.with_raw_response.create(**request_json)
+
+                completion_response = raw_response.parse()
+                completion_response_headers = raw_response.headers
+                completion_status_code = raw_response.status_code
+                if "content-encoding" in completion_response_headers:
+                    completion_response_headers.pop(
+                        "content-encoding")  # Remove "br" that will be changed by this aiproxy
+
+                # Handling response from API
+                if request_json.get("stream"):
+                    async def process_stream(stream: AsyncContentStream) -> AsyncGenerator[str, None]:
+                        # Async content generator
+                        try:
+                            async for chunk in stream:
+                                self.access_logger_queue.put(self.stream_response_item_class(
+                                    request_id=request_id,
+                                    chunk_json=chunk.model_dump()
+                                ))
+                                if chunk:
+                                    yield chunk.model_dump_json()
+
+                        finally:
+                            # Close client after reading stream
+                            await async_client.close()
+
+                            # Response log
+                            now = time.time()
+                            self.access_logger_queue.put(self.stream_response_item_class(
+                                request_id=request_id,
+                                response_headers=completion_response_headers,
+                                duration=now - start_time,
+                                duration_api=now - start_time_api,
+                                request_json=request_json,
+                                status_code=completion_status_code
+                            ))
+
+                    return self.return_response_with_headers(EventSourceResponse(
+                        process_stream(completion_response),
+                        headers=completion_response_headers
+                    ), request_id)
+
+                else:
+                    # Close client immediately
+                    await async_client.close()
+
+                    duration_api = time.time() - start_time_api
+
+                    # Filter response
+                    completion_response = await self.filter_response(request_id, completion_response)
+
+                    # Response log
+                    self.access_logger_queue.put(self.response_item_class(
+                        request_id=request_id,
+                        response_json=completion_response.model_dump(),
+                        response_headers=completion_response_headers,
+                        duration=time.time() - start_time,
+                        duration_api=duration_api,
+                        status_code=completion_status_code
+                    ))
+
+                    return self.return_response_with_headers(JSONResponse(
+                        content=completion_response.model_dump(),
+                        headers=completion_response_headers
+                    ), request_id)
+
+            # Error handlers
+            except RequestFilterException as rfex:
+                logger.error(f"Request filter error: {rfex}\n{traceback.format_exc()}")
+
+                resp_json = {
+                    "error": {"message": rfex.message, "type": "request_filter_error", "param": None, "code": None}}
+
+                # Error log
+                self.access_logger_queue.put(self.error_item_class(
+                    request_id=request_id,
+                    exception=rfex,
+                    traceback_info=traceback.format_exc(),
+                    response_json=resp_json,
+                    status_code=rfex.status_code
+                ))
+
+                return self.return_response_with_headers(JSONResponse(resp_json, status_code=rfex.status_code),
+                                                         request_id)
+
+            except ResponseFilterException as rfex:
+                logger.error(f"Response filter error: {rfex}\n{traceback.format_exc()}")
+
+                resp_json = {
+                    "error": {"message": rfex.message, "type": "response_filter_error", "param": None, "code": None}}
+
+                # Error log
+                self.access_logger_queue.put(self.error_item_class(
+                    request_id=request_id,
+                    exception=rfex,
+                    traceback_info=traceback.format_exc(),
+                    response_json=resp_json,
+                    status_code=rfex.status_code
+                ))
+
+                return self.return_response_with_headers(JSONResponse(resp_json, status_code=rfex.status_code),
+                                                         request_id)
+
+            except (APIStatusError, APIResponseValidationError) as status_err:
+                logger.error(f"APIStatusError from ChatGPT: {status_err}\n{traceback.format_exc()}")
+
+                # Error log
+                try:
+                    resp_json = status_err.response.json()
+                except:
+                    resp_json = str(status_err.response.content)
+
+                self.access_logger_queue.put(self.error_item_class(
+                    request_id=request_id,
+                    exception=status_err,
+                    traceback_info=traceback.format_exc(),
+                    response_json=resp_json,
+                    status_code=status_err.status_code
+                ))
+
+                return self.return_response_with_headers(JSONResponse(resp_json, status_code=status_err.status_code),
+                                                         request_id)
+
+            except APIError as api_err:
+                logger.error(f"APIError from ChatGPT: {api_err}\n{traceback.format_exc()}")
+
+                resp_json = {"error": {"message": api_err.message, "type": api_err.type, "param": api_err.param,
+                                       "code": api_err.code}}
+
+                # Error log
+                self.access_logger_queue.put(self.error_item_class(
+                    request_id=request_id,
+                    exception=api_err,
+                    traceback_info=traceback.format_exc(),
+                    response_json=resp_json,
+                    status_code=502
+                ))
+
+                return self.return_response_with_headers(JSONResponse(resp_json, status_code=502), request_id)
+
+            except OpenAIError as oai_err:
+                logger.error(f"OpenAIError: {oai_err}\n{traceback.format_exc()}")
+
+                resp_json = {"error": {"message": str(oai_err), "type": "openai_error", "param": None, "code": None}}
+
+                # Error log
+                self.access_logger_queue.put(self.error_item_class(
+                    request_id=request_id,
+                    exception=oai_err,
+                    traceback_info=traceback.format_exc(),
+                    response_json=resp_json,
+                    status_code=502
+                ))
+
+                return self.return_response_with_headers(JSONResponse(resp_json, status_code=502), request_id)
+
+            except Exception as ex:
+                logger.error(f"Error at server: {ex}\n{traceback.format_exc()}")
+
+                resp_json = {"error": {"message": "Proxy error", "type": "proxy_error", "param": None, "code": None}}
+
+                # Error log
+                self.access_logger_queue.put(self.error_item_class(
+                    request_id=request_id,
+                    exception=ex,
+                    traceback_info=traceback.format_exc(),
+                    response_json=resp_json,
+                    status_code=502
+                ))
+
+                return self.return_response_with_headers(JSONResponse(resp_json, status_code=502), request_id)

aiproxy/proxy.py

@@ -0,0 +1,74 @@
+from abc import ABC, abstractmethod
+import logging
+from typing import List, Union
+from fastapi import FastAPI
+from fastapi.responses import Response
+from aiproxy.queueclient import QueueClientBase
+
+
+logger = logging.getLogger(__name__)
+
+
+# Classes for filter
+class RequestFilterBase(ABC):
+    @abstractmethod
+    async def filter(self, request_id: str, request_json: dict, request_headers: dict) -> Union[str, None]:
+        ...
+
+
+class ResponseFilterBase(ABC):
+    @abstractmethod
+    async def filter(self, request_id: str, response_json: dict) -> Union[dict, None]:
+        ...
+
+
+class FilterException(Exception):
+    def __init__(self, message: str, status_code: int = 400) -> None:
+        self.message = message
+        self.status_code = status_code
+
+
+class RequestFilterException(FilterException): ...
+
+
+class ResponseFilterException(FilterException): ...
+
+
+class ProxyBase(ABC):
+    def __init__(
+        self,
+        *,
+        request_filters: List[RequestFilterBase] = None,
+        response_filters: List[ResponseFilterBase] = None,
+        access_logger_queue: QueueClientBase
+    ):
+        # Filters
+        self.request_filters = request_filters or []
+        self.response_filters = response_filters or []
+
+        # Access logger queue
+        self.access_logger_queue = access_logger_queue
+
+    def add_filter(self, filter: Union[RequestFilterBase, ResponseFilterBase]):
+        if isinstance(filter, RequestFilterBase):
+            self.request_filters.append(filter)
+            logger.info(f"request filter: {filter.__class__.__name__}")
+        elif isinstance(filter, ResponseFilterBase):
+            self.response_filters.append(filter)
+            logger.info(f"response filter: {filter.__class__.__name__}")
+        else:
+            logger.warning(f"Invalid filter: {filter.__class__.__name__}")
+
+    def add_response_headers(self, response: Response, request_id: str, headers: dict = None):
+        response.headers["X-AIProxy-Request-Id"] = request_id
+        if headers:
+            for k, v in headers.items():
+                response.headers[k] = v
+
+    @abstractmethod
+    def add_route(self, app: FastAPI, base_url: str):
+        ...
+
+    # @abstractmethod
+    # def add_completion_route(self, app: FastAPI, base_url: str):
+    #     ...

aiproxy/queueclient.py

@@ -0,0 +1,51 @@
+from abc import ABC, abstractmethod
+import json
+from queue import Queue
+from typing import Iterator
+
+
+class QueueItemBase(ABC):
+    def to_dict(self) -> dict:
+        d = self.__dict__
+        d["type"] = self.__class__.__name__
+        return d
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict())
+
+    @classmethod
+    def from_dict(cls, d: dict):
+        _d = d.copy()
+        del _d["type"]
+        return cls(**_d)
+
+    @classmethod
+    def from_json(cls, json_str: str):
+        return cls.from_dict(json.loads(json_str))
+
+
+class QueueClientBase(ABC):
+    dequeue_interval = 0.5
+
+    @abstractmethod
+    def put(self, item: QueueItemBase):
+        ...
+
+    @abstractmethod
+    def get(self) -> Iterator[QueueItemBase]:
+        ...
+
+
+class DefaultQueueClient(QueueClientBase):
+    def __init__(self) -> None:
+        self.queue = Queue()
+        self.dequeue_interval = 0.5
+
+    def put(self, item: QueueItemBase):
+        self.queue.put(item)
+
+    def get(self) -> Iterator[QueueItemBase]:
+        items = []
+        while not self.queue.empty():
+            items.append(self.queue.get())
+        return iter(items)