Spaces:
Running
Running
Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- .circleci/config.yml +0 -0
- .circleci/requirements.txt +15 -0
- .devcontainer/devcontainer.json +52 -0
- .dockerignore +12 -0
- .env.example +31 -0
- .flake8 +46 -0
- .git-blame-ignore-revs +10 -0
- .gitattributes +263 -35
- .github/FUNDING.yml +13 -0
- .github/ISSUE_TEMPLATE/bug_report.yml +49 -0
- .github/ISSUE_TEMPLATE/config.yml +8 -0
- .github/ISSUE_TEMPLATE/feature_request.yml +42 -0
- .github/actions/helm-oci-chart-releaser/action.yml +77 -0
- .github/dependabot.yaml +10 -0
- .github/deploy-to-aws.png +0 -0
- .github/pull_request_template.md +33 -0
- .github/template.yaml +94 -0
- .github/workflows/auto_update_price_and_context_window.yml +28 -0
- .github/workflows/auto_update_price_and_context_window_file.py +121 -0
- .github/workflows/ghcr_deploy.yml +433 -0
- .github/workflows/ghcr_helm_deploy.yml +67 -0
- .github/workflows/helm_unit_test.yml +27 -0
- .github/workflows/interpret_load_test.py +138 -0
- .github/workflows/label-mlops.yml +17 -0
- .github/workflows/load_test.yml +59 -0
- .github/workflows/locustfile.py +28 -0
- .github/workflows/main.yml +34 -0
- .github/workflows/publish-migrations.yml +206 -0
- .github/workflows/read_pyproject_version.yml +31 -0
- .github/workflows/redeploy_proxy.py +20 -0
- .github/workflows/reset_stable.yml +39 -0
- .github/workflows/results_stats.csv +27 -0
- .github/workflows/stale.yml +20 -0
- .github/workflows/test-linting.yml +57 -0
- .github/workflows/test-litellm.yml +40 -0
- .github/workflows/update_release.py +54 -0
- .pre-commit-config.yaml +40 -0
- AGENTS.md +144 -0
- CONTRIBUTING.md +274 -0
- Dockerfile +78 -0
- LICENSE +26 -0
- Makefile +90 -0
- README.md +441 -12
- ci_cd/baseline_db.py +60 -0
- ci_cd/check_file_length.py +28 -0
- ci_cd/check_files_match.py +32 -0
- ci_cd/publish-proxy-extras.sh +19 -0
- ci_cd/run_migration.py +95 -0
- codecov.yaml +32 -0
- cookbook/Benchmarking_LLMs_by_use_case.ipynb +0 -0
.circleci/config.yml
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
.circleci/requirements.txt
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# used by CI/CD testing
|
| 2 |
+
openai==1.81.0
|
| 3 |
+
python-dotenv
|
| 4 |
+
tiktoken
|
| 5 |
+
importlib_metadata
|
| 6 |
+
cohere
|
| 7 |
+
redis==5.2.1
|
| 8 |
+
redisvl==0.4.1
|
| 9 |
+
anthropic
|
| 10 |
+
orjson==3.10.12 # fast /embedding responses
|
| 11 |
+
pydantic==2.10.2
|
| 12 |
+
google-cloud-aiplatform==1.43.0
|
| 13 |
+
fastapi-sso==0.16.0
|
| 14 |
+
uvloop==0.21.0
|
| 15 |
+
mcp==1.9.3 # for MCP server
|
.devcontainer/devcontainer.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "Python 3.11",
|
| 3 |
+
// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
|
| 4 |
+
"image": "mcr.microsoft.com/devcontainers/python:3.11-bookworm",
|
| 5 |
+
// https://github.com/devcontainers/images/tree/main/src/python
|
| 6 |
+
// https://mcr.microsoft.com/en-us/product/devcontainers/python/tags
|
| 7 |
+
|
| 8 |
+
// "build": {
|
| 9 |
+
// "dockerfile": "Dockerfile",
|
| 10 |
+
// "context": ".."
|
| 11 |
+
// },
|
| 12 |
+
|
| 13 |
+
// Features to add to the dev container. More info: https://containers.dev/features.
|
| 14 |
+
// "features": {},
|
| 15 |
+
|
| 16 |
+
// Configure tool-specific properties.
|
| 17 |
+
"customizations": {
|
| 18 |
+
// Configure properties specific to VS Code.
|
| 19 |
+
"vscode": {
|
| 20 |
+
"settings": {},
|
| 21 |
+
"extensions": [
|
| 22 |
+
"ms-python.python",
|
| 23 |
+
"ms-python.vscode-pylance",
|
| 24 |
+
"GitHub.copilot",
|
| 25 |
+
"GitHub.copilot-chat",
|
| 26 |
+
"ms-python.autopep8"
|
| 27 |
+
]
|
| 28 |
+
}
|
| 29 |
+
},
|
| 30 |
+
|
| 31 |
+
// Use 'forwardPorts' to make a list of ports inside the container available locally.
|
| 32 |
+
"forwardPorts": [4000],
|
| 33 |
+
|
| 34 |
+
"containerEnv": {
|
| 35 |
+
"LITELLM_LOG": "DEBUG"
|
| 36 |
+
},
|
| 37 |
+
|
| 38 |
+
// Use 'portsAttributes' to set default properties for specific forwarded ports.
|
| 39 |
+
// More info: https://containers.dev/implementors/json_reference/#port-attributes
|
| 40 |
+
"portsAttributes": {
|
| 41 |
+
"4000": {
|
| 42 |
+
"label": "LiteLLM Server",
|
| 43 |
+
"onAutoForward": "notify"
|
| 44 |
+
}
|
| 45 |
+
},
|
| 46 |
+
|
| 47 |
+
// More info: https://aka.ms/dev-containers-non-root.
|
| 48 |
+
// "remoteUser": "litellm",
|
| 49 |
+
|
| 50 |
+
// Use 'postCreateCommand' to run commands after the container is created.
|
| 51 |
+
"postCreateCommand": "pipx install poetry && poetry install -E extra_proxy -E proxy"
|
| 52 |
+
}
|
.dockerignore
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
docs
|
| 2 |
+
cookbook
|
| 3 |
+
.circleci
|
| 4 |
+
.github
|
| 5 |
+
tests
|
| 6 |
+
.git
|
| 7 |
+
.github
|
| 8 |
+
.circleci
|
| 9 |
+
.devcontainer
|
| 10 |
+
*.tgz
|
| 11 |
+
log.txt
|
| 12 |
+
docker/Dockerfile.*
|
.env.example
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# OpenAI
|
| 2 |
+
OPENAI_API_KEY = ""
|
| 3 |
+
OPENAI_BASE_URL = ""
|
| 4 |
+
# Cohere
|
| 5 |
+
COHERE_API_KEY = ""
|
| 6 |
+
# OpenRouter
|
| 7 |
+
OR_SITE_URL = ""
|
| 8 |
+
OR_APP_NAME = "LiteLLM Example app"
|
| 9 |
+
OR_API_KEY = ""
|
| 10 |
+
# Azure API base URL
|
| 11 |
+
AZURE_API_BASE = ""
|
| 12 |
+
# Azure API version
|
| 13 |
+
AZURE_API_VERSION = ""
|
| 14 |
+
# Azure API key
|
| 15 |
+
AZURE_API_KEY = ""
|
| 16 |
+
# Replicate
|
| 17 |
+
REPLICATE_API_KEY = ""
|
| 18 |
+
REPLICATE_API_TOKEN = ""
|
| 19 |
+
# Anthropic
|
| 20 |
+
ANTHROPIC_API_KEY = ""
|
| 21 |
+
# Infisical
|
| 22 |
+
INFISICAL_TOKEN = ""
|
| 23 |
+
# Novita AI
|
| 24 |
+
NOVITA_API_KEY = ""
|
| 25 |
+
# INFINITY
|
| 26 |
+
INFINITY_API_KEY = ""
|
| 27 |
+
|
| 28 |
+
# Development Configs
|
| 29 |
+
LITELLM_MASTER_KEY = "sk-1234"
|
| 30 |
+
DATABASE_URL = "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
|
| 31 |
+
STORE_MODEL_IN_DB = "True"
|
.flake8
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[flake8]
|
| 2 |
+
ignore =
|
| 3 |
+
# The following ignores can be removed when formatting using black
|
| 4 |
+
W191,W291,W292,W293,W391,W504
|
| 5 |
+
E101,E111,E114,E116,E117,E121,E122,E123,E124,E125,E126,E127,E128,E129,E131,
|
| 6 |
+
E201,E202,E221,E222,E225,E226,E231,E241,E251,E252,E261,E265,E271,E272,E275,
|
| 7 |
+
E301,E302,E303,E305,E306,
|
| 8 |
+
# line break before binary operator
|
| 9 |
+
W503,
|
| 10 |
+
# inline comment should start with '# '
|
| 11 |
+
E262,
|
| 12 |
+
# too many leading '#' for block comment
|
| 13 |
+
E266,
|
| 14 |
+
# multiple imports on one line
|
| 15 |
+
E401,
|
| 16 |
+
# module level import not at top of file
|
| 17 |
+
E402,
|
| 18 |
+
# Line too long (82 > 79 characters)
|
| 19 |
+
E501,
|
| 20 |
+
# comparison to None should be 'if cond is None:'
|
| 21 |
+
E711,
|
| 22 |
+
# comparison to True should be 'if cond is True:' or 'if cond:'
|
| 23 |
+
E712,
|
| 24 |
+
# do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()`
|
| 25 |
+
E721,
|
| 26 |
+
# do not use bare 'except'
|
| 27 |
+
E722,
|
| 28 |
+
# x is imported but unused
|
| 29 |
+
F401,
|
| 30 |
+
# 'from . import *' used; unable to detect undefined names
|
| 31 |
+
F403,
|
| 32 |
+
# x may be undefined, or defined from star imports:
|
| 33 |
+
F405,
|
| 34 |
+
# f-string is missing placeholders
|
| 35 |
+
F541,
|
| 36 |
+
# dictionary key '' repeated with different values
|
| 37 |
+
F601,
|
| 38 |
+
# redefinition of unused x from line 123
|
| 39 |
+
F811,
|
| 40 |
+
# undefined name x
|
| 41 |
+
F821,
|
| 42 |
+
# local variable x is assigned to but never used
|
| 43 |
+
F841,
|
| 44 |
+
|
| 45 |
+
# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8
|
| 46 |
+
extend-ignore = E203
|
.git-blame-ignore-revs
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Add the commit hash of any commit you want to ignore in `git blame` here.
|
| 2 |
+
# One commit hash per line.
|
| 3 |
+
#
|
| 4 |
+
# The GitHub Blame UI will use this file automatically!
|
| 5 |
+
#
|
| 6 |
+
# Run this command to always ignore formatting commits in `git blame`
|
| 7 |
+
# git config blame.ignoreRevsFile .git-blame-ignore-revs
|
| 8 |
+
|
| 9 |
+
# Update pydantic code to fix warnings (GH-3600)
|
| 10 |
+
876840e9957bc7e9f7d6a2b58c4d7c53dad16481
|
.gitattributes
CHANGED
|
@@ -1,35 +1,263 @@
|
|
| 1 |
-
*.
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.ipynb linguist-vendoredcookbook/codellama-server/imgs/code-output.png filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
cookbook/codellama-server/imgs/promptlayer_logging.png filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
cookbook/logging_observability/litellm_proxy_langfuse.png filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
deploy/azure_resource_manager/azure_marketplace.zip filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
deploy/charts/litellm-helm/charts/postgresql-14.3.1.tgz filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
deploy/charts/litellm-helm/charts/redis-18.19.1.tgz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
dist/litellm-1.57.6.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
docs/my-website/img/10_instance_proxy.png filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
docs/my-website/img/1_instance_proxy.png filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
docs/my-website/img/2_instance_proxy.png filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
docs/my-website/img/add_internal_user.png filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
docs/my-website/img/admin_ui_2.png filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
docs/my-website/img/admin_ui_disabled.png filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
docs/my-website/img/admin_ui_spend.png filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
docs/my-website/img/admin_ui_viewer.png filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
docs/my-website/img/alerting_metadata.png filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
docs/my-website/img/alt_dashboard.png filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
docs/my-website/img/aporia_post.png filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
docs/my-website/img/aporia_pre.png filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
docs/my-website/img/aporia_projs.png filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
docs/my-website/img/argilla.png filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
docs/my-website/img/arize.png filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
docs/my-website/img/athina_dashboard.png filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
docs/my-website/img/auto_prompt_caching.png filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
docs/my-website/img/azure_blob.png filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
docs/my-website/img/basic_litellm.gif filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
docs/my-website/img/bench_llm.png filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
docs/my-website/img/callback_api.png filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
docs/my-website/img/cloud_run0.png filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
docs/my-website/img/cloud_run1.png filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
docs/my-website/img/cloud_run2.png filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
docs/my-website/img/cloud_run3.png filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
docs/my-website/img/compare_llms.png filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
docs/my-website/img/control_model_access_jwt.png filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
docs/my-website/img/create_budget_modal.png filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
docs/my-website/img/create_key_in_team.gif filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
docs/my-website/img/create_key_in_team_oweb.gif filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
docs/my-website/img/create_team_gif_good.gif filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
docs/my-website/img/custom_prompt_management.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
docs/my-website/img/custom_root_path.png filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
docs/my-website/img/custom_swagger.png filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
docs/my-website/img/dash_output.png filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
docs/my-website/img/dashboard_log.png filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
docs/my-website/img/dd_small1.png filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
docs/my-website/img/debug_langfuse.png filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
docs/my-website/img/debug_sso.png filter=lfs diff=lfs merge=lfs -text
|
| 47 |
+
docs/my-website/img/deepeval_dashboard.png filter=lfs diff=lfs merge=lfs -text
|
| 48 |
+
docs/my-website/img/deepeval_visible_trace.png filter=lfs diff=lfs merge=lfs -text
|
| 49 |
+
docs/my-website/img/delete_spend_logs.jpg filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
docs/my-website/img/elastic_otel.png filter=lfs diff=lfs merge=lfs -text
|
| 51 |
+
docs/my-website/img/email_2.png filter=lfs diff=lfs merge=lfs -text
|
| 52 |
+
docs/my-website/img/email_2_0.png filter=lfs diff=lfs merge=lfs -text
|
| 53 |
+
docs/my-website/img/email_event_1.png filter=lfs diff=lfs merge=lfs -text
|
| 54 |
+
docs/my-website/img/email_event_2.png filter=lfs diff=lfs merge=lfs -text
|
| 55 |
+
docs/my-website/img/email_notifs.png filter=lfs diff=lfs merge=lfs -text
|
| 56 |
+
docs/my-website/img/end_user_enforcement.png filter=lfs diff=lfs merge=lfs -text
|
| 57 |
+
docs/my-website/img/enterprise_vs_oss.png filter=lfs diff=lfs merge=lfs -text
|
| 58 |
+
docs/my-website/img/entra_create_team.png filter=lfs diff=lfs merge=lfs -text
|
| 59 |
+
docs/my-website/img/files_api_graphic.png filter=lfs diff=lfs merge=lfs -text
|
| 60 |
+
docs/my-website/img/gcp_acc_2.png filter=lfs diff=lfs merge=lfs -text
|
| 61 |
+
docs/my-website/img/gcp_acc_3.png filter=lfs diff=lfs merge=lfs -text
|
| 62 |
+
docs/my-website/img/gcs_bucket.png filter=lfs diff=lfs merge=lfs -text
|
| 63 |
+
docs/my-website/img/gd_fail.png filter=lfs diff=lfs merge=lfs -text
|
| 64 |
+
docs/my-website/img/gd_success.png filter=lfs diff=lfs merge=lfs -text
|
| 65 |
+
docs/my-website/img/gemini_context_caching.png filter=lfs diff=lfs merge=lfs -text
|
| 66 |
+
docs/my-website/img/gemini_realtime.png filter=lfs diff=lfs merge=lfs -text
|
| 67 |
+
docs/my-website/img/google_oauth2.png filter=lfs diff=lfs merge=lfs -text
|
| 68 |
+
docs/my-website/img/google_redirect.png filter=lfs diff=lfs merge=lfs -text
|
| 69 |
+
docs/my-website/img/grafana_1.png filter=lfs diff=lfs merge=lfs -text
|
| 70 |
+
docs/my-website/img/grafana_2.png filter=lfs diff=lfs merge=lfs -text
|
| 71 |
+
docs/my-website/img/grafana_3.png filter=lfs diff=lfs merge=lfs -text
|
| 72 |
+
docs/my-website/img/hcorp.png filter=lfs diff=lfs merge=lfs -text
|
| 73 |
+
docs/my-website/img/hcorp_create_virtual_key.png filter=lfs diff=lfs merge=lfs -text
|
| 74 |
+
docs/my-website/img/hcorp_virtual_key.png filter=lfs diff=lfs merge=lfs -text
|
| 75 |
+
docs/my-website/img/hf_filter_inference_providers.png filter=lfs diff=lfs merge=lfs -text
|
| 76 |
+
docs/my-website/img/hf_inference_endpoint.png filter=lfs diff=lfs merge=lfs -text
|
| 77 |
+
docs/my-website/img/hosted_debugger_usage_page.png filter=lfs diff=lfs merge=lfs -text
|
| 78 |
+
docs/my-website/img/instances_vs_rps.png filter=lfs diff=lfs merge=lfs -text
|
| 79 |
+
docs/my-website/img/invitation_link.png filter=lfs diff=lfs merge=lfs -text
|
| 80 |
+
docs/my-website/img/kb.png filter=lfs diff=lfs merge=lfs -text
|
| 81 |
+
docs/my-website/img/kb_2.png filter=lfs diff=lfs merge=lfs -text
|
| 82 |
+
docs/my-website/img/kb_3.png filter=lfs diff=lfs merge=lfs -text
|
| 83 |
+
docs/my-website/img/kb_4.png filter=lfs diff=lfs merge=lfs -text
|
| 84 |
+
docs/my-website/img/key_delete.png filter=lfs diff=lfs merge=lfs -text
|
| 85 |
+
docs/my-website/img/key_email.png filter=lfs diff=lfs merge=lfs -text
|
| 86 |
+
docs/my-website/img/key_email_2.png filter=lfs diff=lfs merge=lfs -text
|
| 87 |
+
docs/my-website/img/lago.jpeg filter=lfs diff=lfs merge=lfs -text
|
| 88 |
+
docs/my-website/img/lago_2.png filter=lfs diff=lfs merge=lfs -text
|
| 89 |
+
docs/my-website/img/langfuse-litellm-ui.png filter=lfs diff=lfs merge=lfs -text
|
| 90 |
+
docs/my-website/img/langfuse.png filter=lfs diff=lfs merge=lfs -text
|
| 91 |
+
docs/my-website/img/langfuse_prmpt_mgmt.png filter=lfs diff=lfs merge=lfs -text
|
| 92 |
+
docs/my-website/img/langfuse_small.png filter=lfs diff=lfs merge=lfs -text
|
| 93 |
+
docs/my-website/img/langsmith.png filter=lfs diff=lfs merge=lfs -text
|
| 94 |
+
docs/my-website/img/langsmith_new.png filter=lfs diff=lfs merge=lfs -text
|
| 95 |
+
docs/my-website/img/litellm_adk.png filter=lfs diff=lfs merge=lfs -text
|
| 96 |
+
docs/my-website/img/litellm_codex.gif filter=lfs diff=lfs merge=lfs -text
|
| 97 |
+
docs/my-website/img/litellm_create_team.gif filter=lfs diff=lfs merge=lfs -text
|
| 98 |
+
docs/my-website/img/litellm_hosted_ui_add_models.png filter=lfs diff=lfs merge=lfs -text
|
| 99 |
+
docs/my-website/img/litellm_hosted_ui_create_key.png filter=lfs diff=lfs merge=lfs -text
|
| 100 |
+
docs/my-website/img/litellm_hosted_ui_router.png filter=lfs diff=lfs merge=lfs -text
|
| 101 |
+
docs/my-website/img/litellm_hosted_usage_dashboard.png filter=lfs diff=lfs merge=lfs -text
|
| 102 |
+
docs/my-website/img/litellm_load_test.png filter=lfs diff=lfs merge=lfs -text
|
| 103 |
+
docs/my-website/img/litellm_mcp.png filter=lfs diff=lfs merge=lfs -text
|
| 104 |
+
docs/my-website/img/litellm_setup_openweb.gif filter=lfs diff=lfs merge=lfs -text
|
| 105 |
+
docs/my-website/img/litellm_streamlit_playground.png filter=lfs diff=lfs merge=lfs -text
|
| 106 |
+
docs/my-website/img/litellm_thinking_openweb.gif filter=lfs diff=lfs merge=lfs -text
|
| 107 |
+
docs/my-website/img/litellm_ui_3.gif filter=lfs diff=lfs merge=lfs -text
|
| 108 |
+
docs/my-website/img/litellm_ui_create_key.png filter=lfs diff=lfs merge=lfs -text
|
| 109 |
+
docs/my-website/img/litellm_ui_login.png filter=lfs diff=lfs merge=lfs -text
|
| 110 |
+
docs/my-website/img/litellm_user_heirarchy.png filter=lfs diff=lfs merge=lfs -text
|
| 111 |
+
docs/my-website/img/literalai.png filter=lfs diff=lfs merge=lfs -text
|
| 112 |
+
docs/my-website/img/locust.png filter=lfs diff=lfs merge=lfs -text
|
| 113 |
+
docs/my-website/img/locust_load_test.png filter=lfs diff=lfs merge=lfs -text
|
| 114 |
+
docs/my-website/img/locust_load_test1.png filter=lfs diff=lfs merge=lfs -text
|
| 115 |
+
docs/my-website/img/locust_load_test2.png filter=lfs diff=lfs merge=lfs -text
|
| 116 |
+
docs/my-website/img/locust_load_test2_setup.png filter=lfs diff=lfs merge=lfs -text
|
| 117 |
+
docs/my-website/img/logfire.png filter=lfs diff=lfs merge=lfs -text
|
| 118 |
+
docs/my-website/img/lunary-trace.png filter=lfs diff=lfs merge=lfs -text
|
| 119 |
+
docs/my-website/img/managed_files_arch.png filter=lfs diff=lfs merge=lfs -text
|
| 120 |
+
docs/my-website/img/max_budget_for_internal_users.png filter=lfs diff=lfs merge=lfs -text
|
| 121 |
+
docs/my-website/img/mcp_2.png filter=lfs diff=lfs merge=lfs -text
|
| 122 |
+
docs/my-website/img/message_redaction_logging.png filter=lfs diff=lfs merge=lfs -text
|
| 123 |
+
docs/my-website/img/message_redaction_spend_logs.png filter=lfs diff=lfs merge=lfs -text
|
| 124 |
+
docs/my-website/img/mlflow_tracing.png filter=lfs diff=lfs merge=lfs -text
|
| 125 |
+
docs/my-website/img/model_hub.png filter=lfs diff=lfs merge=lfs -text
|
| 126 |
+
docs/my-website/img/ms_teams_alerting.png filter=lfs diff=lfs merge=lfs -text
|
| 127 |
+
docs/my-website/img/msft_default_settings.png filter=lfs diff=lfs merge=lfs -text
|
| 128 |
+
docs/my-website/img/msft_enterprise_app.png filter=lfs diff=lfs merge=lfs -text
|
| 129 |
+
docs/my-website/img/msft_enterprise_assign_group.png filter=lfs diff=lfs merge=lfs -text
|
| 130 |
+
docs/my-website/img/msft_enterprise_select_group.png filter=lfs diff=lfs merge=lfs -text
|
| 131 |
+
docs/my-website/img/msft_member_1.png filter=lfs diff=lfs merge=lfs -text
|
| 132 |
+
docs/my-website/img/msft_member_2.png filter=lfs diff=lfs merge=lfs -text
|
| 133 |
+
docs/my-website/img/msft_member_3.png filter=lfs diff=lfs merge=lfs -text
|
| 134 |
+
docs/my-website/img/msft_sso_sign_in.png filter=lfs diff=lfs merge=lfs -text
|
| 135 |
+
docs/my-website/img/multiple_deployments.png filter=lfs diff=lfs merge=lfs -text
|
| 136 |
+
docs/my-website/img/new_user_email.png filter=lfs diff=lfs merge=lfs -text
|
| 137 |
+
docs/my-website/img/okta_callback_url.png filter=lfs diff=lfs merge=lfs -text
|
| 138 |
+
docs/my-website/img/openmeter.png filter=lfs diff=lfs merge=lfs -text
|
| 139 |
+
docs/my-website/img/openmeter_img_2.png filter=lfs diff=lfs merge=lfs -text
|
| 140 |
+
docs/my-website/img/opik.png filter=lfs diff=lfs merge=lfs -text
|
| 141 |
+
docs/my-website/img/otel_debug_trace.png filter=lfs diff=lfs merge=lfs -text
|
| 142 |
+
docs/my-website/img/otel_parent.png filter=lfs diff=lfs merge=lfs -text
|
| 143 |
+
docs/my-website/img/pagerduty_fail.png filter=lfs diff=lfs merge=lfs -text
|
| 144 |
+
docs/my-website/img/pagerduty_hanging.png filter=lfs diff=lfs merge=lfs -text
|
| 145 |
+
docs/my-website/img/perf_imp.png filter=lfs diff=lfs merge=lfs -text
|
| 146 |
+
docs/my-website/img/pii_masking_v2.png filter=lfs diff=lfs merge=lfs -text
|
| 147 |
+
docs/my-website/img/presidio_1.png filter=lfs diff=lfs merge=lfs -text
|
| 148 |
+
docs/my-website/img/presidio_2.png filter=lfs diff=lfs merge=lfs -text
|
| 149 |
+
docs/my-website/img/presidio_3.png filter=lfs diff=lfs merge=lfs -text
|
| 150 |
+
docs/my-website/img/presidio_4.png filter=lfs diff=lfs merge=lfs -text
|
| 151 |
+
docs/my-website/img/presidio_5.png filter=lfs diff=lfs merge=lfs -text
|
| 152 |
+
docs/my-website/img/presidio_screenshot.png filter=lfs diff=lfs merge=lfs -text
|
| 153 |
+
docs/my-website/img/prevent_deadlocks.jpg filter=lfs diff=lfs merge=lfs -text
|
| 154 |
+
docs/my-website/img/prompt_management_architecture_doc.png filter=lfs diff=lfs merge=lfs -text
|
| 155 |
+
docs/my-website/img/promptlayer.png filter=lfs diff=lfs merge=lfs -text
|
| 156 |
+
docs/my-website/img/proxy_langfuse.png filter=lfs diff=lfs merge=lfs -text
|
| 157 |
+
docs/my-website/img/raw_request_log.png filter=lfs diff=lfs merge=lfs -text
|
| 158 |
+
docs/my-website/img/raw_response_headers.png filter=lfs diff=lfs merge=lfs -text
|
| 159 |
+
docs/my-website/img/realtime_api.png filter=lfs diff=lfs merge=lfs -text
|
| 160 |
+
docs/my-website/img/release_notes/anthropic_thinking.jpg filter=lfs diff=lfs merge=lfs -text
|
| 161 |
+
docs/my-website/img/release_notes/bedrock_kb.png filter=lfs diff=lfs merge=lfs -text
|
| 162 |
+
docs/my-website/img/release_notes/chat_metrics.png filter=lfs diff=lfs merge=lfs -text
|
| 163 |
+
docs/my-website/img/release_notes/credentials.jpg filter=lfs diff=lfs merge=lfs -text
|
| 164 |
+
docs/my-website/img/release_notes/error_logs.jpg filter=lfs diff=lfs merge=lfs -text
|
| 165 |
+
docs/my-website/img/release_notes/lb_batch.png filter=lfs diff=lfs merge=lfs -text
|
| 166 |
+
docs/my-website/img/release_notes/litellm_test_connection.gif filter=lfs diff=lfs merge=lfs -text
|
| 167 |
+
docs/my-website/img/release_notes/mcp_ui.png filter=lfs diff=lfs merge=lfs -text
|
| 168 |
+
docs/my-website/img/release_notes/multi_instance_rate_limits_v3.jpg filter=lfs diff=lfs merge=lfs -text
|
| 169 |
+
docs/my-website/img/release_notes/new_activity_tab.png filter=lfs diff=lfs merge=lfs -text
|
| 170 |
+
docs/my-website/img/release_notes/new_tag_usage.png filter=lfs diff=lfs merge=lfs -text
|
| 171 |
+
docs/my-website/img/release_notes/new_team_usage.png filter=lfs diff=lfs merge=lfs -text
|
| 172 |
+
docs/my-website/img/release_notes/new_team_usage_highlight.jpg filter=lfs diff=lfs merge=lfs -text
|
| 173 |
+
docs/my-website/img/release_notes/spend_by_model.jpg filter=lfs diff=lfs merge=lfs -text
|
| 174 |
+
docs/my-website/img/release_notes/tag_management.png filter=lfs diff=lfs merge=lfs -text
|
| 175 |
+
docs/my-website/img/release_notes/team_filters.png filter=lfs diff=lfs merge=lfs -text
|
| 176 |
+
docs/my-website/img/release_notes/ui_audit_log.png filter=lfs diff=lfs merge=lfs -text
|
| 177 |
+
docs/my-website/img/release_notes/ui_format.png filter=lfs diff=lfs merge=lfs -text
|
| 178 |
+
docs/my-website/img/release_notes/ui_logs.png filter=lfs diff=lfs merge=lfs -text
|
| 179 |
+
docs/my-website/img/release_notes/ui_model.png filter=lfs diff=lfs merge=lfs -text
|
| 180 |
+
docs/my-website/img/release_notes/ui_responses_lb.png filter=lfs diff=lfs merge=lfs -text
|
| 181 |
+
docs/my-website/img/release_notes/ui_search_users.png filter=lfs diff=lfs merge=lfs -text
|
| 182 |
+
docs/my-website/img/release_notes/unified_responses_api_rn.png filter=lfs diff=lfs merge=lfs -text
|
| 183 |
+
docs/my-website/img/release_notes/user_filters.png filter=lfs diff=lfs merge=lfs -text
|
| 184 |
+
docs/my-website/img/release_notes/v1632_release.jpg filter=lfs diff=lfs merge=lfs -text
|
| 185 |
+
docs/my-website/img/release_notes/v1_messages_perf.png filter=lfs diff=lfs merge=lfs -text
|
| 186 |
+
docs/my-website/img/render1.png filter=lfs diff=lfs merge=lfs -text
|
| 187 |
+
docs/my-website/img/render2.png filter=lfs diff=lfs merge=lfs -text
|
| 188 |
+
docs/my-website/img/response_cost_img.png filter=lfs diff=lfs merge=lfs -text
|
| 189 |
+
docs/my-website/img/sagemaker_deploy.png filter=lfs diff=lfs merge=lfs -text
|
| 190 |
+
docs/my-website/img/sagemaker_domain.png filter=lfs diff=lfs merge=lfs -text
|
| 191 |
+
docs/my-website/img/sagemaker_endpoint.png filter=lfs diff=lfs merge=lfs -text
|
| 192 |
+
docs/my-website/img/sagemaker_jumpstart.png filter=lfs diff=lfs merge=lfs -text
|
| 193 |
+
docs/my-website/img/scim_0.png filter=lfs diff=lfs merge=lfs -text
|
| 194 |
+
docs/my-website/img/scim_1.png filter=lfs diff=lfs merge=lfs -text
|
| 195 |
+
docs/my-website/img/scim_2.png filter=lfs diff=lfs merge=lfs -text
|
| 196 |
+
docs/my-website/img/scim_3.png filter=lfs diff=lfs merge=lfs -text
|
| 197 |
+
docs/my-website/img/scim_4.png filter=lfs diff=lfs merge=lfs -text
|
| 198 |
+
docs/my-website/img/sentry.png filter=lfs diff=lfs merge=lfs -text
|
| 199 |
+
docs/my-website/img/slack.png filter=lfs diff=lfs merge=lfs -text
|
| 200 |
+
docs/my-website/img/spend_log_deletion_multi_pod.jpg filter=lfs diff=lfs merge=lfs -text
|
| 201 |
+
docs/my-website/img/spend_log_deletion_working.png filter=lfs diff=lfs merge=lfs -text
|
| 202 |
+
docs/my-website/img/spend_logs_table.png filter=lfs diff=lfs merge=lfs -text
|
| 203 |
+
docs/my-website/img/spend_per_user.png filter=lfs diff=lfs merge=lfs -text
|
| 204 |
+
docs/my-website/img/swagger.png filter=lfs diff=lfs merge=lfs -text
|
| 205 |
+
docs/my-website/img/tag_create.png filter=lfs diff=lfs merge=lfs -text
|
| 206 |
+
docs/my-website/img/tag_invalid.png filter=lfs diff=lfs merge=lfs -text
|
| 207 |
+
docs/my-website/img/tag_valid.png filter=lfs diff=lfs merge=lfs -text
|
| 208 |
+
docs/my-website/img/test_key_budget.gif filter=lfs diff=lfs merge=lfs -text
|
| 209 |
+
docs/my-website/img/test_python_server_2.png filter=lfs diff=lfs merge=lfs -text
|
| 210 |
+
docs/my-website/img/traceloop_dash.png filter=lfs diff=lfs merge=lfs -text
|
| 211 |
+
docs/my-website/img/ui_3.gif filter=lfs diff=lfs merge=lfs -text
|
| 212 |
+
docs/my-website/img/ui_add_cred_2.png filter=lfs diff=lfs merge=lfs -text
|
| 213 |
+
docs/my-website/img/ui_auto_prompt_caching.png filter=lfs diff=lfs merge=lfs -text
|
| 214 |
+
docs/my-website/img/ui_clean_login.png filter=lfs diff=lfs merge=lfs -text
|
| 215 |
+
docs/my-website/img/ui_cred_3.png filter=lfs diff=lfs merge=lfs -text
|
| 216 |
+
docs/my-website/img/ui_cred_4.png filter=lfs diff=lfs merge=lfs -text
|
| 217 |
+
docs/my-website/img/ui_cred_add.png filter=lfs diff=lfs merge=lfs -text
|
| 218 |
+
docs/my-website/img/ui_invite_user.png filter=lfs diff=lfs merge=lfs -text
|
| 219 |
+
docs/my-website/img/ui_request_logs.png filter=lfs diff=lfs merge=lfs -text
|
| 220 |
+
docs/my-website/img/ui_request_logs_content.png filter=lfs diff=lfs merge=lfs -text
|
| 221 |
+
docs/my-website/img/ui_self_serve_create_key.png filter=lfs diff=lfs merge=lfs -text
|
| 222 |
+
docs/my-website/img/ui_session_logs.png filter=lfs diff=lfs merge=lfs -text
|
| 223 |
+
docs/my-website/img/ui_usage.png filter=lfs diff=lfs merge=lfs -text
|
| 224 |
+
docs/my-website/img/use_model_cred.png filter=lfs diff=lfs merge=lfs -text
|
| 225 |
+
docs/my-website/img/wandb.png filter=lfs diff=lfs merge=lfs -text
|
| 226 |
+
docs/my-website/static/img/docusaurus-social-card.png filter=lfs diff=lfs merge=lfs -text
|
| 227 |
+
enterprise/dist/litellm_enterprise-0.1.1.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 228 |
+
enterprise/dist/litellm_enterprise-0.1.2.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 229 |
+
enterprise/dist/litellm_enterprise-0.1.3.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 230 |
+
enterprise/dist/litellm_enterprise-0.1.4.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 231 |
+
enterprise/dist/litellm_enterprise-0.1.5.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 232 |
+
enterprise/dist/litellm_enterprise-0.1.6.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 233 |
+
enterprise/dist/litellm_enterprise-0.1.7.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 234 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.0.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 235 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.1.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 236 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.12.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 237 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.14.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 238 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.15.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 239 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.17.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 240 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.18.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 241 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.19.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 242 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.2.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 243 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.20.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 244 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.21.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 245 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.3.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 246 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.4.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 247 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.7.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 248 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.8.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 249 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.1.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 250 |
+
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.2.tar.gz filter=lfs diff=lfs merge=lfs -text
|
| 251 |
+
tests/gettysburg.wav filter=lfs diff=lfs merge=lfs -text
|
| 252 |
+
tests/image_gen_tests/ishaan_github.png filter=lfs diff=lfs merge=lfs -text
|
| 253 |
+
tests/image_gen_tests/litellm_site.png filter=lfs diff=lfs merge=lfs -text
|
| 254 |
+
tests/image_gen_tests/test_image.png filter=lfs diff=lfs merge=lfs -text
|
| 255 |
+
tests/llm_translation/duck.png filter=lfs diff=lfs merge=lfs -text
|
| 256 |
+
tests/llm_translation/gettysburg.wav filter=lfs diff=lfs merge=lfs -text
|
| 257 |
+
tests/llm_translation/guinea.png filter=lfs diff=lfs merge=lfs -text
|
| 258 |
+
tests/local_testing/gettysburg.wav filter=lfs diff=lfs merge=lfs -text
|
| 259 |
+
tests/local_testing/speech_vertex.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 260 |
+
tests/logging_callback_tests/gettysburg.wav filter=lfs diff=lfs merge=lfs -text
|
| 261 |
+
tests/proxy_unit_tests/gettysburg.wav filter=lfs diff=lfs merge=lfs -text
|
| 262 |
+
tests/proxy_unit_tests/speech_vertex.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 263 |
+
tests/router_unit_tests/gettysburg.wav filter=lfs diff=lfs merge=lfs -text
|
.github/FUNDING.yml
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# These are supported funding model platforms
|
| 2 |
+
|
| 3 |
+
github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
|
| 4 |
+
patreon: # Replace with a single Patreon username
|
| 5 |
+
open_collective: # Replace with a single Open Collective username
|
| 6 |
+
ko_fi: # Replace with a single Ko-fi username
|
| 7 |
+
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
| 8 |
+
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
| 9 |
+
liberapay: # Replace with a single Liberapay username
|
| 10 |
+
issuehunt: # Replace with a single IssueHunt username
|
| 11 |
+
otechie: # Replace with a single Otechie username
|
| 12 |
+
lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
|
| 13 |
+
custom: https://buy.stripe.com/9AQ03Kd3P91o0Q8bIS
|
.github/ISSUE_TEMPLATE/bug_report.yml
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Bug Report
|
| 2 |
+
description: File a bug report
|
| 3 |
+
title: "[Bug]: "
|
| 4 |
+
labels: ["bug"]
|
| 5 |
+
body:
|
| 6 |
+
- type: markdown
|
| 7 |
+
attributes:
|
| 8 |
+
value: |
|
| 9 |
+
Thanks for taking the time to fill out this bug report!
|
| 10 |
+
- type: textarea
|
| 11 |
+
id: what-happened
|
| 12 |
+
attributes:
|
| 13 |
+
label: What happened?
|
| 14 |
+
description: Also tell us, what did you expect to happen?
|
| 15 |
+
placeholder: Tell us what you see!
|
| 16 |
+
value: "A bug happened!"
|
| 17 |
+
validations:
|
| 18 |
+
required: true
|
| 19 |
+
- type: textarea
|
| 20 |
+
id: logs
|
| 21 |
+
attributes:
|
| 22 |
+
label: Relevant log output
|
| 23 |
+
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
|
| 24 |
+
render: shell
|
| 25 |
+
- type: dropdown
|
| 26 |
+
id: ml-ops-team
|
| 27 |
+
attributes:
|
| 28 |
+
label: Are you a ML Ops Team?
|
| 29 |
+
description: This helps us prioritize your requests correctly
|
| 30 |
+
options:
|
| 31 |
+
- "No"
|
| 32 |
+
- "Yes"
|
| 33 |
+
validations:
|
| 34 |
+
required: true
|
| 35 |
+
- type: input
|
| 36 |
+
id: version
|
| 37 |
+
attributes:
|
| 38 |
+
label: What LiteLLM version are you on ?
|
| 39 |
+
placeholder: v1.53.1
|
| 40 |
+
validations:
|
| 41 |
+
required: true
|
| 42 |
+
- type: input
|
| 43 |
+
id: contact
|
| 44 |
+
attributes:
|
| 45 |
+
label: Twitter / LinkedIn details
|
| 46 |
+
description: We announce new features on Twitter + LinkedIn. If this issue leads to an announcement, and you'd like a mention, we'll gladly shout you out!
|
| 47 |
+
placeholder: ex. @krrish_dh / https://www.linkedin.com/in/krish-d/
|
| 48 |
+
validations:
|
| 49 |
+
required: false
|
.github/ISSUE_TEMPLATE/config.yml
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
blank_issues_enabled: true
|
| 2 |
+
contact_links:
|
| 3 |
+
- name: Schedule Demo
|
| 4 |
+
url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat
|
| 5 |
+
about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM
|
| 6 |
+
- name: Discord
|
| 7 |
+
url: https://discord.com/invite/wuPM9dRgDw
|
| 8 |
+
about: Join 250+ LiteLLM community members!
|
.github/ISSUE_TEMPLATE/feature_request.yml
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: 🚀 Feature Request
|
| 2 |
+
description: Submit a proposal/request for a new LiteLLM feature.
|
| 3 |
+
title: "[Feature]: "
|
| 4 |
+
labels: ["enhancement"]
|
| 5 |
+
body:
|
| 6 |
+
- type: markdown
|
| 7 |
+
attributes:
|
| 8 |
+
value: |
|
| 9 |
+
Thanks for making LiteLLM better!
|
| 10 |
+
- type: textarea
|
| 11 |
+
id: the-feature
|
| 12 |
+
attributes:
|
| 13 |
+
label: The Feature
|
| 14 |
+
description: A clear and concise description of the feature proposal
|
| 15 |
+
placeholder: Tell us what you want!
|
| 16 |
+
validations:
|
| 17 |
+
required: true
|
| 18 |
+
- type: textarea
|
| 19 |
+
id: motivation
|
| 20 |
+
attributes:
|
| 21 |
+
label: Motivation, pitch
|
| 22 |
+
description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too.
|
| 23 |
+
validations:
|
| 24 |
+
required: true
|
| 25 |
+
- type: dropdown
|
| 26 |
+
id: hiring-interest
|
| 27 |
+
attributes:
|
| 28 |
+
label: LiteLLM is hiring a founding backend engineer, are you interested in joining us and shipping to all our users?
|
| 29 |
+
description: If yes, apply here - https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer
|
| 30 |
+
options:
|
| 31 |
+
- "No"
|
| 32 |
+
- "Yes"
|
| 33 |
+
validations:
|
| 34 |
+
required: true
|
| 35 |
+
- type: input
|
| 36 |
+
id: contact
|
| 37 |
+
attributes:
|
| 38 |
+
label: Twitter / LinkedIn details
|
| 39 |
+
description: We announce new features on Twitter + LinkedIn. When this is announced, and you'd like a mention, we'll gladly shout you out!
|
| 40 |
+
placeholder: ex. @krrish_dh / https://www.linkedin.com/in/krish-d/
|
| 41 |
+
validations:
|
| 42 |
+
required: false
|
.github/actions/helm-oci-chart-releaser/action.yml
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Helm OCI Chart Releaser
|
| 2 |
+
description: Push Helm charts to OCI-based (Docker) registries
|
| 3 |
+
author: sergeyshaykhullin
|
| 4 |
+
branding:
|
| 5 |
+
color: yellow
|
| 6 |
+
icon: upload-cloud
|
| 7 |
+
inputs:
|
| 8 |
+
name:
|
| 9 |
+
required: true
|
| 10 |
+
description: Chart name
|
| 11 |
+
repository:
|
| 12 |
+
required: true
|
| 13 |
+
description: Chart repository name
|
| 14 |
+
tag:
|
| 15 |
+
required: true
|
| 16 |
+
description: Chart version
|
| 17 |
+
app_version:
|
| 18 |
+
required: true
|
| 19 |
+
description: App version
|
| 20 |
+
path:
|
| 21 |
+
required: false
|
| 22 |
+
description: Chart path (Default 'charts/{name}')
|
| 23 |
+
registry:
|
| 24 |
+
required: true
|
| 25 |
+
description: OCI registry
|
| 26 |
+
registry_username:
|
| 27 |
+
required: true
|
| 28 |
+
description: OCI registry username
|
| 29 |
+
registry_password:
|
| 30 |
+
required: true
|
| 31 |
+
description: OCI registry password
|
| 32 |
+
update_dependencies:
|
| 33 |
+
required: false
|
| 34 |
+
default: 'false'
|
| 35 |
+
description: Update chart dependencies before packaging (Default 'false')
|
| 36 |
+
outputs:
|
| 37 |
+
image:
|
| 38 |
+
value: ${{ steps.output.outputs.image }}
|
| 39 |
+
description: Chart image (Default '{registry}/{repository}/{image}:{tag}')
|
| 40 |
+
runs:
|
| 41 |
+
using: composite
|
| 42 |
+
steps:
|
| 43 |
+
- name: Helm | Login
|
| 44 |
+
shell: bash
|
| 45 |
+
run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }}
|
| 46 |
+
env:
|
| 47 |
+
HELM_EXPERIMENTAL_OCI: '1'
|
| 48 |
+
|
| 49 |
+
- name: Helm | Dependency
|
| 50 |
+
if: inputs.update_dependencies == 'true'
|
| 51 |
+
shell: bash
|
| 52 |
+
run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
|
| 53 |
+
env:
|
| 54 |
+
HELM_EXPERIMENTAL_OCI: '1'
|
| 55 |
+
|
| 56 |
+
- name: Helm | Package
|
| 57 |
+
shell: bash
|
| 58 |
+
run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }}
|
| 59 |
+
env:
|
| 60 |
+
HELM_EXPERIMENTAL_OCI: '1'
|
| 61 |
+
|
| 62 |
+
- name: Helm | Push
|
| 63 |
+
shell: bash
|
| 64 |
+
run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }}
|
| 65 |
+
env:
|
| 66 |
+
HELM_EXPERIMENTAL_OCI: '1'
|
| 67 |
+
|
| 68 |
+
- name: Helm | Logout
|
| 69 |
+
shell: bash
|
| 70 |
+
run: helm registry logout ${{ inputs.registry }}
|
| 71 |
+
env:
|
| 72 |
+
HELM_EXPERIMENTAL_OCI: '1'
|
| 73 |
+
|
| 74 |
+
- name: Helm | Output
|
| 75 |
+
id: output
|
| 76 |
+
shell: bash
|
| 77 |
+
run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
|
.github/dependabot.yaml
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version: 2
|
| 2 |
+
updates:
|
| 3 |
+
- package-ecosystem: "github-actions"
|
| 4 |
+
directory: "/"
|
| 5 |
+
schedule:
|
| 6 |
+
interval: "daily"
|
| 7 |
+
groups:
|
| 8 |
+
github-actions:
|
| 9 |
+
patterns:
|
| 10 |
+
- "*"
|
.github/deploy-to-aws.png
ADDED
|
.github/pull_request_template.md
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## Title
|
| 2 |
+
|
| 3 |
+
<!-- e.g. "Implement user authentication feature" -->
|
| 4 |
+
|
| 5 |
+
## Relevant issues
|
| 6 |
+
|
| 7 |
+
<!-- e.g. "Fixes #000" -->
|
| 8 |
+
|
| 9 |
+
## Pre-Submission checklist
|
| 10 |
+
|
| 11 |
+
**Please complete all items before asking a LiteLLM maintainer to review your PR**
|
| 12 |
+
|
| 13 |
+
- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
|
| 14 |
+
- [ ] I have added a screenshot of my new test passing locally
|
| 15 |
+
- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
|
| 16 |
+
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
## Type
|
| 20 |
+
|
| 21 |
+
<!-- Select the type of Pull Request -->
|
| 22 |
+
<!-- Keep only the necessary ones -->
|
| 23 |
+
|
| 24 |
+
🆕 New Feature
|
| 25 |
+
🐛 Bug Fix
|
| 26 |
+
🧹 Refactoring
|
| 27 |
+
📖 Documentation
|
| 28 |
+
🚄 Infrastructure
|
| 29 |
+
✅ Test
|
| 30 |
+
|
| 31 |
+
## Changes
|
| 32 |
+
|
| 33 |
+
|
.github/template.yaml
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
AWSTemplateFormatVersion: '2010-09-09'
|
| 2 |
+
Transform: AWS::Serverless-2016-10-31
|
| 3 |
+
Description: >
|
| 4 |
+
llmlite-service
|
| 5 |
+
|
| 6 |
+
SAM Template for llmlite-service
|
| 7 |
+
|
| 8 |
+
# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst
|
| 9 |
+
Globals:
|
| 10 |
+
Function:
|
| 11 |
+
Timeout: 600
|
| 12 |
+
MemorySize: 128
|
| 13 |
+
Environment:
|
| 14 |
+
Variables:
|
| 15 |
+
WORKER_CONFIG: !Ref WorkerConfigParameter
|
| 16 |
+
|
| 17 |
+
Parameters:
|
| 18 |
+
AliasParameter:
|
| 19 |
+
Type: String
|
| 20 |
+
Default: live
|
| 21 |
+
WorkerConfigParameter:
|
| 22 |
+
Type: String
|
| 23 |
+
Description: Sample environment variable
|
| 24 |
+
Default: '{"model": null, "alias": null, "api_base": null, "api_version": "2023-07-01-preview", "debug": false, "temperature": null, "max_tokens": null, "request_timeout": 600, "max_budget": null, "telemetry": true, "drop_params": false, "add_function_to_prompt": false, "headers": null, "save": false, "config": null, "use_queue": false}'
|
| 25 |
+
|
| 26 |
+
Resources:
|
| 27 |
+
MyUrlFunctionPermissions:
|
| 28 |
+
Type: AWS::Lambda::Permission
|
| 29 |
+
Properties:
|
| 30 |
+
FunctionName: !Ref URL
|
| 31 |
+
Action: lambda:InvokeFunctionUrl
|
| 32 |
+
Principal: "*"
|
| 33 |
+
FunctionUrlAuthType: NONE
|
| 34 |
+
|
| 35 |
+
Function:
|
| 36 |
+
Type: AWS::Serverless::Function
|
| 37 |
+
Properties:
|
| 38 |
+
FunctionName: !Sub "${AWS::StackName}-function"
|
| 39 |
+
CodeUri: "./litellm"
|
| 40 |
+
Handler: proxy/lambda.handler
|
| 41 |
+
Runtime: python3.11
|
| 42 |
+
AutoPublishAlias: !Ref AliasParameter
|
| 43 |
+
Architectures:
|
| 44 |
+
- x86_64
|
| 45 |
+
DeploymentPreference:
|
| 46 |
+
Type: AllAtOnce
|
| 47 |
+
Alarms:
|
| 48 |
+
- !Ref NewVersionErrorMetricGreaterThanZeroAlarm
|
| 49 |
+
|
| 50 |
+
NewVersionErrorMetricGreaterThanZeroAlarm:
|
| 51 |
+
Type: "AWS::CloudWatch::Alarm"
|
| 52 |
+
Properties:
|
| 53 |
+
AlarmDescription: Lambda Function Error > 0
|
| 54 |
+
ComparisonOperator: GreaterThanThreshold
|
| 55 |
+
Dimensions:
|
| 56 |
+
- Name: Resource
|
| 57 |
+
Value: !Sub "${Function}:live"
|
| 58 |
+
- Name: FunctionName
|
| 59 |
+
Value: !Ref Function
|
| 60 |
+
- Name: ExecutedVersion
|
| 61 |
+
Value: !GetAtt Function.Version.Version
|
| 62 |
+
EvaluationPeriods: 1
|
| 63 |
+
Unit: Count
|
| 64 |
+
MetricName: Errors
|
| 65 |
+
Namespace: AWS/Lambda
|
| 66 |
+
Period: 60
|
| 67 |
+
Statistic: Sum
|
| 68 |
+
Threshold: 0
|
| 69 |
+
|
| 70 |
+
URL:
|
| 71 |
+
Type: AWS::Lambda::Url
|
| 72 |
+
DependsOn: FunctionAliaslive
|
| 73 |
+
Properties:
|
| 74 |
+
AuthType: NONE
|
| 75 |
+
Qualifier: live
|
| 76 |
+
TargetFunctionArn: !GetAtt Function.Arn
|
| 77 |
+
|
| 78 |
+
Outputs:
|
| 79 |
+
FunctionARN:
|
| 80 |
+
Description: "Lambda Function ARN"
|
| 81 |
+
Value: !GetAtt Function.Arn
|
| 82 |
+
|
| 83 |
+
FunctionUrl:
|
| 84 |
+
Description: "Lambda Function URL Endpoint"
|
| 85 |
+
Value:
|
| 86 |
+
Fn::GetAtt: URL.FunctionUrl
|
| 87 |
+
|
| 88 |
+
FunctionVersion:
|
| 89 |
+
Description: "Lambda Function Version"
|
| 90 |
+
Value: !GetAtt Function.Version.Version
|
| 91 |
+
|
| 92 |
+
FunctionNewAlarmARN:
|
| 93 |
+
Description: "Lambda Function New Alarm ARN"
|
| 94 |
+
Value: !GetAtt NewVersionErrorMetricGreaterThanZeroAlarm.Arn
|
.github/workflows/auto_update_price_and_context_window.yml
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Updates model_prices_and_context_window.json and Create Pull Request
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
schedule:
|
| 5 |
+
- cron: "0 0 * * 0" # Run every Sundays at midnight
|
| 6 |
+
#- cron: "0 0 * * *" # Run daily at midnight
|
| 7 |
+
|
| 8 |
+
jobs:
|
| 9 |
+
auto_update_price_and_context_window:
|
| 10 |
+
runs-on: ubuntu-latest
|
| 11 |
+
steps:
|
| 12 |
+
- uses: actions/checkout@v3
|
| 13 |
+
- name: Install Dependencies
|
| 14 |
+
run: |
|
| 15 |
+
pip install aiohttp
|
| 16 |
+
- name: Update JSON Data
|
| 17 |
+
run: |
|
| 18 |
+
python ".github/workflows/auto_update_price_and_context_window_file.py"
|
| 19 |
+
- name: Create Pull Request
|
| 20 |
+
run: |
|
| 21 |
+
git add model_prices_and_context_window.json
|
| 22 |
+
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
|
| 23 |
+
gh pr create --title "Update model_prices_and_context_window.json file" \
|
| 24 |
+
--body "Automated update for model_prices_and_context_window.json" \
|
| 25 |
+
--head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \
|
| 26 |
+
--base main
|
| 27 |
+
env:
|
| 28 |
+
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
.github/workflows/auto_update_price_and_context_window_file.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import aiohttp
|
| 3 |
+
import json
|
| 4 |
+
|
| 5 |
+
# Asynchronously fetch data from a given URL
|
| 6 |
+
async def fetch_data(url):
|
| 7 |
+
try:
|
| 8 |
+
# Create an asynchronous session
|
| 9 |
+
async with aiohttp.ClientSession() as session:
|
| 10 |
+
# Send a GET request to the URL
|
| 11 |
+
async with session.get(url) as resp:
|
| 12 |
+
# Raise an error if the response status is not OK
|
| 13 |
+
resp.raise_for_status()
|
| 14 |
+
# Parse the response JSON
|
| 15 |
+
resp_json = await resp.json()
|
| 16 |
+
print("Fetch the data from URL.")
|
| 17 |
+
# Return the 'data' field from the JSON response
|
| 18 |
+
return resp_json['data']
|
| 19 |
+
except Exception as e:
|
| 20 |
+
# Print an error message if fetching data fails
|
| 21 |
+
print("Error fetching data from URL:", e)
|
| 22 |
+
return None
|
| 23 |
+
|
| 24 |
+
# Synchronize local data with remote data
|
| 25 |
+
def sync_local_data_with_remote(local_data, remote_data):
|
| 26 |
+
# Update existing keys in local_data with values from remote_data
|
| 27 |
+
for key in (set(local_data) & set(remote_data)):
|
| 28 |
+
local_data[key].update(remote_data[key])
|
| 29 |
+
|
| 30 |
+
# Add new keys from remote_data to local_data
|
| 31 |
+
for key in (set(remote_data) - set(local_data)):
|
| 32 |
+
local_data[key] = remote_data[key]
|
| 33 |
+
|
| 34 |
+
# Write data to the json file
|
| 35 |
+
def write_to_file(file_path, data):
|
| 36 |
+
try:
|
| 37 |
+
# Open the file in write mode
|
| 38 |
+
with open(file_path, "w") as file:
|
| 39 |
+
# Dump the data as JSON into the file
|
| 40 |
+
json.dump(data, file, indent=4)
|
| 41 |
+
print("Values updated successfully.")
|
| 42 |
+
except Exception as e:
|
| 43 |
+
# Print an error message if writing to file fails
|
| 44 |
+
print("Error updating JSON file:", e)
|
| 45 |
+
|
| 46 |
+
# Update the existing models and add the missing models
|
| 47 |
+
def transform_remote_data(data):
|
| 48 |
+
transformed = {}
|
| 49 |
+
for row in data:
|
| 50 |
+
# Add the fields 'max_tokens' and 'input_cost_per_token'
|
| 51 |
+
obj = {
|
| 52 |
+
"max_tokens": row["context_length"],
|
| 53 |
+
"input_cost_per_token": float(row["pricing"]["prompt"]),
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
# Add 'max_output_tokens' as a field if it is not None
|
| 57 |
+
if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None:
|
| 58 |
+
obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"])
|
| 59 |
+
|
| 60 |
+
# Add the field 'output_cost_per_token'
|
| 61 |
+
obj.update({
|
| 62 |
+
"output_cost_per_token": float(row["pricing"]["completion"]),
|
| 63 |
+
})
|
| 64 |
+
|
| 65 |
+
# Add field 'input_cost_per_image' if it exists and is non-zero
|
| 66 |
+
if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0:
|
| 67 |
+
obj['input_cost_per_image'] = float(row["pricing"]["image"])
|
| 68 |
+
|
| 69 |
+
# Add the fields 'litellm_provider' and 'mode'
|
| 70 |
+
obj.update({
|
| 71 |
+
"litellm_provider": "openrouter",
|
| 72 |
+
"mode": "chat"
|
| 73 |
+
})
|
| 74 |
+
|
| 75 |
+
# Add the 'supports_vision' field if the modality is 'multimodal'
|
| 76 |
+
if row.get('architecture', {}).get('modality') == 'multimodal':
|
| 77 |
+
obj['supports_vision'] = True
|
| 78 |
+
|
| 79 |
+
# Use a composite key to store the transformed object
|
| 80 |
+
transformed[f'openrouter/{row["id"]}'] = obj
|
| 81 |
+
|
| 82 |
+
return transformed
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
# Load local data from a specified file
|
| 86 |
+
def load_local_data(file_path):
|
| 87 |
+
try:
|
| 88 |
+
# Open the file in read mode
|
| 89 |
+
with open(file_path, "r") as file:
|
| 90 |
+
# Load and return the JSON data
|
| 91 |
+
return json.load(file)
|
| 92 |
+
except FileNotFoundError:
|
| 93 |
+
# Print an error message if the file is not found
|
| 94 |
+
print("File not found:", file_path)
|
| 95 |
+
return None
|
| 96 |
+
except json.JSONDecodeError as e:
|
| 97 |
+
# Print an error message if JSON decoding fails
|
| 98 |
+
print("Error decoding JSON:", e)
|
| 99 |
+
return None
|
| 100 |
+
|
| 101 |
+
def main():
|
| 102 |
+
local_file_path = "model_prices_and_context_window.json" # Path to the local data file
|
| 103 |
+
url = "https://openrouter.ai/api/v1/models" # URL to fetch remote data
|
| 104 |
+
|
| 105 |
+
# Load local data from file
|
| 106 |
+
local_data = load_local_data(local_file_path)
|
| 107 |
+
# Fetch remote data asynchronously
|
| 108 |
+
remote_data = asyncio.run(fetch_data(url))
|
| 109 |
+
# Transform the fetched remote data
|
| 110 |
+
remote_data = transform_remote_data(remote_data)
|
| 111 |
+
|
| 112 |
+
# If both local and remote data are available, synchronize and save
|
| 113 |
+
if local_data and remote_data:
|
| 114 |
+
sync_local_data_with_remote(local_data, remote_data)
|
| 115 |
+
write_to_file(local_file_path, local_data)
|
| 116 |
+
else:
|
| 117 |
+
print("Failed to fetch model data from either local file or URL.")
|
| 118 |
+
|
| 119 |
+
# Entry point of the script
|
| 120 |
+
if __name__ == "__main__":
|
| 121 |
+
main()
|
.github/workflows/ghcr_deploy.yml
ADDED
|
@@ -0,0 +1,433 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
|
| 2 |
+
name: Build, Publish LiteLLM Docker Image. New Release
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch:
|
| 5 |
+
inputs:
|
| 6 |
+
tag:
|
| 7 |
+
description: "The tag version you want to build"
|
| 8 |
+
release_type:
|
| 9 |
+
description: "The release type you want to build. Can be 'latest', 'stable', 'dev'"
|
| 10 |
+
type: string
|
| 11 |
+
default: "latest"
|
| 12 |
+
commit_hash:
|
| 13 |
+
description: "Commit hash"
|
| 14 |
+
required: true
|
| 15 |
+
|
| 16 |
+
# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
|
| 17 |
+
env:
|
| 18 |
+
REGISTRY: ghcr.io
|
| 19 |
+
IMAGE_NAME: ${{ github.repository }}
|
| 20 |
+
CHART_NAME: litellm-helm
|
| 21 |
+
|
| 22 |
+
# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
|
| 23 |
+
jobs:
|
| 24 |
+
# print commit hash, tag, and release type
|
| 25 |
+
print:
|
| 26 |
+
runs-on: ubuntu-latest
|
| 27 |
+
steps:
|
| 28 |
+
- run: |
|
| 29 |
+
echo "Commit hash: ${{ github.event.inputs.commit_hash }}"
|
| 30 |
+
echo "Tag: ${{ github.event.inputs.tag }}"
|
| 31 |
+
echo "Release type: ${{ github.event.inputs.release_type }}"
|
| 32 |
+
docker-hub-deploy:
|
| 33 |
+
if: github.repository == 'BerriAI/litellm'
|
| 34 |
+
runs-on: ubuntu-latest
|
| 35 |
+
steps:
|
| 36 |
+
-
|
| 37 |
+
name: Checkout
|
| 38 |
+
uses: actions/checkout@v4
|
| 39 |
+
with:
|
| 40 |
+
ref: ${{ github.event.inputs.commit_hash }}
|
| 41 |
+
-
|
| 42 |
+
name: Set up QEMU
|
| 43 |
+
uses: docker/setup-qemu-action@v3
|
| 44 |
+
-
|
| 45 |
+
name: Set up Docker Buildx
|
| 46 |
+
uses: docker/setup-buildx-action@v3
|
| 47 |
+
-
|
| 48 |
+
name: Login to Docker Hub
|
| 49 |
+
uses: docker/login-action@v3
|
| 50 |
+
with:
|
| 51 |
+
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
| 52 |
+
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
| 53 |
+
-
|
| 54 |
+
name: Build and push
|
| 55 |
+
uses: docker/build-push-action@v5
|
| 56 |
+
with:
|
| 57 |
+
context: .
|
| 58 |
+
push: true
|
| 59 |
+
tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }}
|
| 60 |
+
-
|
| 61 |
+
name: Build and push litellm-database image
|
| 62 |
+
uses: docker/build-push-action@v5
|
| 63 |
+
with:
|
| 64 |
+
context: .
|
| 65 |
+
push: true
|
| 66 |
+
file: ./docker/Dockerfile.database
|
| 67 |
+
tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }}
|
| 68 |
+
-
|
| 69 |
+
name: Build and push litellm-spend-logs image
|
| 70 |
+
uses: docker/build-push-action@v5
|
| 71 |
+
with:
|
| 72 |
+
context: .
|
| 73 |
+
push: true
|
| 74 |
+
file: ./litellm-js/spend-logs/Dockerfile
|
| 75 |
+
tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }}
|
| 76 |
+
|
| 77 |
+
build-and-push-image:
|
| 78 |
+
runs-on: ubuntu-latest
|
| 79 |
+
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
|
| 80 |
+
permissions:
|
| 81 |
+
contents: read
|
| 82 |
+
packages: write
|
| 83 |
+
steps:
|
| 84 |
+
- name: Checkout repository
|
| 85 |
+
uses: actions/checkout@v4
|
| 86 |
+
with:
|
| 87 |
+
ref: ${{ github.event.inputs.commit_hash }}
|
| 88 |
+
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
|
| 89 |
+
- name: Log in to the Container registry
|
| 90 |
+
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
|
| 91 |
+
with:
|
| 92 |
+
registry: ${{ env.REGISTRY }}
|
| 93 |
+
username: ${{ github.actor }}
|
| 94 |
+
password: ${{ secrets.GITHUB_TOKEN }}
|
| 95 |
+
# This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
|
| 96 |
+
- name: Extract metadata (tags, labels) for Docker
|
| 97 |
+
id: meta
|
| 98 |
+
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
|
| 99 |
+
with:
|
| 100 |
+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
| 101 |
+
# Configure multi platform Docker builds
|
| 102 |
+
- name: Set up QEMU
|
| 103 |
+
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
|
| 104 |
+
- name: Set up Docker Buildx
|
| 105 |
+
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
|
| 106 |
+
# This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
|
| 107 |
+
# It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
|
| 108 |
+
# It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
|
| 109 |
+
- name: Build and push Docker image
|
| 110 |
+
uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8
|
| 111 |
+
with:
|
| 112 |
+
context: .
|
| 113 |
+
push: true
|
| 114 |
+
tags: |
|
| 115 |
+
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
|
| 116 |
+
${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }}
|
| 117 |
+
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
|
| 118 |
+
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }},
|
| 119 |
+
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
|
| 120 |
+
labels: ${{ steps.meta.outputs.labels }}
|
| 121 |
+
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
|
| 122 |
+
|
| 123 |
+
build-and-push-image-ee:
|
| 124 |
+
runs-on: ubuntu-latest
|
| 125 |
+
permissions:
|
| 126 |
+
contents: read
|
| 127 |
+
packages: write
|
| 128 |
+
steps:
|
| 129 |
+
- name: Checkout repository
|
| 130 |
+
uses: actions/checkout@v4
|
| 131 |
+
with:
|
| 132 |
+
ref: ${{ github.event.inputs.commit_hash }}
|
| 133 |
+
|
| 134 |
+
- name: Log in to the Container registry
|
| 135 |
+
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
|
| 136 |
+
with:
|
| 137 |
+
registry: ${{ env.REGISTRY }}
|
| 138 |
+
username: ${{ github.actor }}
|
| 139 |
+
password: ${{ secrets.GITHUB_TOKEN }}
|
| 140 |
+
|
| 141 |
+
- name: Extract metadata (tags, labels) for EE Dockerfile
|
| 142 |
+
id: meta-ee
|
| 143 |
+
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
|
| 144 |
+
with:
|
| 145 |
+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
|
| 146 |
+
# Configure multi platform Docker builds
|
| 147 |
+
- name: Set up QEMU
|
| 148 |
+
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
|
| 149 |
+
- name: Set up Docker Buildx
|
| 150 |
+
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
|
| 151 |
+
|
| 152 |
+
- name: Build and push EE Docker image
|
| 153 |
+
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
|
| 154 |
+
with:
|
| 155 |
+
context: .
|
| 156 |
+
file: Dockerfile
|
| 157 |
+
push: true
|
| 158 |
+
tags: |
|
| 159 |
+
${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
|
| 160 |
+
${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }}
|
| 161 |
+
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
|
| 162 |
+
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }}
|
| 163 |
+
labels: ${{ steps.meta-ee.outputs.labels }}
|
| 164 |
+
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
|
| 165 |
+
|
| 166 |
+
build-and-push-image-database:
|
| 167 |
+
runs-on: ubuntu-latest
|
| 168 |
+
permissions:
|
| 169 |
+
contents: read
|
| 170 |
+
packages: write
|
| 171 |
+
steps:
|
| 172 |
+
- name: Checkout repository
|
| 173 |
+
uses: actions/checkout@v4
|
| 174 |
+
with:
|
| 175 |
+
ref: ${{ github.event.inputs.commit_hash }}
|
| 176 |
+
|
| 177 |
+
- name: Log in to the Container registry
|
| 178 |
+
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
|
| 179 |
+
with:
|
| 180 |
+
registry: ${{ env.REGISTRY }}
|
| 181 |
+
username: ${{ github.actor }}
|
| 182 |
+
password: ${{ secrets.GITHUB_TOKEN }}
|
| 183 |
+
|
| 184 |
+
- name: Extract metadata (tags, labels) for database Dockerfile
|
| 185 |
+
id: meta-database
|
| 186 |
+
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
|
| 187 |
+
with:
|
| 188 |
+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-database
|
| 189 |
+
# Configure multi platform Docker builds
|
| 190 |
+
- name: Set up QEMU
|
| 191 |
+
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
|
| 192 |
+
- name: Set up Docker Buildx
|
| 193 |
+
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
|
| 194 |
+
|
| 195 |
+
- name: Build and push Database Docker image
|
| 196 |
+
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
|
| 197 |
+
with:
|
| 198 |
+
context: .
|
| 199 |
+
file: ./docker/Dockerfile.database
|
| 200 |
+
push: true
|
| 201 |
+
tags: |
|
| 202 |
+
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
|
| 203 |
+
${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
|
| 204 |
+
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
|
| 205 |
+
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }}
|
| 206 |
+
labels: ${{ steps.meta-database.outputs.labels }}
|
| 207 |
+
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
|
| 208 |
+
|
| 209 |
+
build-and-push-image-non_root:
|
| 210 |
+
runs-on: ubuntu-latest
|
| 211 |
+
permissions:
|
| 212 |
+
contents: read
|
| 213 |
+
packages: write
|
| 214 |
+
steps:
|
| 215 |
+
- name: Checkout repository
|
| 216 |
+
uses: actions/checkout@v4
|
| 217 |
+
with:
|
| 218 |
+
ref: ${{ github.event.inputs.commit_hash }}
|
| 219 |
+
|
| 220 |
+
- name: Log in to the Container registry
|
| 221 |
+
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
|
| 222 |
+
with:
|
| 223 |
+
registry: ${{ env.REGISTRY }}
|
| 224 |
+
username: ${{ github.actor }}
|
| 225 |
+
password: ${{ secrets.GITHUB_TOKEN }}
|
| 226 |
+
|
| 227 |
+
- name: Extract metadata (tags, labels) for non_root Dockerfile
|
| 228 |
+
id: meta-non_root
|
| 229 |
+
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
|
| 230 |
+
with:
|
| 231 |
+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-non_root
|
| 232 |
+
# Configure multi platform Docker builds
|
| 233 |
+
- name: Set up QEMU
|
| 234 |
+
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
|
| 235 |
+
- name: Set up Docker Buildx
|
| 236 |
+
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
|
| 237 |
+
|
| 238 |
+
- name: Build and push non_root Docker image
|
| 239 |
+
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
|
| 240 |
+
with:
|
| 241 |
+
context: .
|
| 242 |
+
file: ./docker/Dockerfile.non_root
|
| 243 |
+
push: true
|
| 244 |
+
tags: |
|
| 245 |
+
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
|
| 246 |
+
${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
|
| 247 |
+
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
|
| 248 |
+
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }}
|
| 249 |
+
labels: ${{ steps.meta-non_root.outputs.labels }}
|
| 250 |
+
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
|
| 251 |
+
|
| 252 |
+
build-and-push-image-spend-logs:
|
| 253 |
+
runs-on: ubuntu-latest
|
| 254 |
+
permissions:
|
| 255 |
+
contents: read
|
| 256 |
+
packages: write
|
| 257 |
+
steps:
|
| 258 |
+
- name: Checkout repository
|
| 259 |
+
uses: actions/checkout@v4
|
| 260 |
+
with:
|
| 261 |
+
ref: ${{ github.event.inputs.commit_hash }}
|
| 262 |
+
|
| 263 |
+
- name: Log in to the Container registry
|
| 264 |
+
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
|
| 265 |
+
with:
|
| 266 |
+
registry: ${{ env.REGISTRY }}
|
| 267 |
+
username: ${{ github.actor }}
|
| 268 |
+
password: ${{ secrets.GITHUB_TOKEN }}
|
| 269 |
+
|
| 270 |
+
- name: Extract metadata (tags, labels) for spend-logs Dockerfile
|
| 271 |
+
id: meta-spend-logs
|
| 272 |
+
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
|
| 273 |
+
with:
|
| 274 |
+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs
|
| 275 |
+
# Configure multi platform Docker builds
|
| 276 |
+
- name: Set up QEMU
|
| 277 |
+
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
|
| 278 |
+
- name: Set up Docker Buildx
|
| 279 |
+
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
|
| 280 |
+
|
| 281 |
+
- name: Build and push Database Docker image
|
| 282 |
+
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
|
| 283 |
+
with:
|
| 284 |
+
context: .
|
| 285 |
+
file: ./litellm-js/spend-logs/Dockerfile
|
| 286 |
+
push: true
|
| 287 |
+
tags: |
|
| 288 |
+
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
|
| 289 |
+
${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
|
| 290 |
+
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
|
| 291 |
+
${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
|
| 292 |
+
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
|
| 293 |
+
|
| 294 |
+
build-and-push-helm-chart:
|
| 295 |
+
if: github.event.inputs.release_type != 'dev'
|
| 296 |
+
needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
|
| 297 |
+
runs-on: ubuntu-latest
|
| 298 |
+
steps:
|
| 299 |
+
- name: Checkout repository
|
| 300 |
+
uses: actions/checkout@v4
|
| 301 |
+
with:
|
| 302 |
+
fetch-depth: 0
|
| 303 |
+
|
| 304 |
+
- name: Log in to the Container registry
|
| 305 |
+
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
|
| 306 |
+
with:
|
| 307 |
+
registry: ${{ env.REGISTRY }}
|
| 308 |
+
username: ${{ github.actor }}
|
| 309 |
+
password: ${{ secrets.GITHUB_TOKEN }}
|
| 310 |
+
|
| 311 |
+
- name: lowercase github.repository_owner
|
| 312 |
+
run: |
|
| 313 |
+
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
|
| 314 |
+
|
| 315 |
+
- name: Get LiteLLM Latest Tag
|
| 316 |
+
id: current_app_tag
|
| 317 |
+
shell: bash
|
| 318 |
+
run: |
|
| 319 |
+
LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0)
|
| 320 |
+
if [ -z "${LATEST_TAG}" ]; then
|
| 321 |
+
echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT
|
| 322 |
+
else
|
| 323 |
+
echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT
|
| 324 |
+
fi
|
| 325 |
+
|
| 326 |
+
- name: Get last published chart version
|
| 327 |
+
id: current_version
|
| 328 |
+
shell: bash
|
| 329 |
+
run: |
|
| 330 |
+
CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
|
| 331 |
+
if [ -z "${CHART_LIST}" ]; then
|
| 332 |
+
echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
|
| 333 |
+
else
|
| 334 |
+
printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
|
| 335 |
+
fi
|
| 336 |
+
env:
|
| 337 |
+
HELM_EXPERIMENTAL_OCI: '1'
|
| 338 |
+
|
| 339 |
+
# Automatically update the helm chart version one "patch" level
|
| 340 |
+
- name: Bump release version
|
| 341 |
+
id: bump_version
|
| 342 |
+
uses: christian-draeger/[email protected]
|
| 343 |
+
with:
|
| 344 |
+
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
|
| 345 |
+
version-fragment: 'bug'
|
| 346 |
+
|
| 347 |
+
- uses: ./.github/actions/helm-oci-chart-releaser
|
| 348 |
+
with:
|
| 349 |
+
name: ${{ env.CHART_NAME }}
|
| 350 |
+
repository: ${{ env.REPO_OWNER }}
|
| 351 |
+
tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
|
| 352 |
+
app_version: ${{ steps.current_app_tag.outputs.latest_tag }}
|
| 353 |
+
path: deploy/charts/${{ env.CHART_NAME }}
|
| 354 |
+
registry: ${{ env.REGISTRY }}
|
| 355 |
+
registry_username: ${{ github.actor }}
|
| 356 |
+
registry_password: ${{ secrets.GITHUB_TOKEN }}
|
| 357 |
+
update_dependencies: true
|
| 358 |
+
|
| 359 |
+
release:
|
| 360 |
+
name: "New LiteLLM Release"
|
| 361 |
+
needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
|
| 362 |
+
|
| 363 |
+
runs-on: "ubuntu-latest"
|
| 364 |
+
|
| 365 |
+
steps:
|
| 366 |
+
- name: Display version
|
| 367 |
+
run: echo "Current version is ${{ github.event.inputs.tag }}"
|
| 368 |
+
- name: "Set Release Tag"
|
| 369 |
+
run: echo "RELEASE_TAG=${{ github.event.inputs.tag }}" >> $GITHUB_ENV
|
| 370 |
+
- name: Display release tag
|
| 371 |
+
run: echo "RELEASE_TAG is $RELEASE_TAG"
|
| 372 |
+
- name: "Create release"
|
| 373 |
+
uses: "actions/github-script@v6"
|
| 374 |
+
with:
|
| 375 |
+
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
| 376 |
+
script: |
|
| 377 |
+
const commitHash = "${{ github.event.inputs.commit_hash}}";
|
| 378 |
+
console.log("Commit Hash:", commitHash); // Add this line for debugging
|
| 379 |
+
try {
|
| 380 |
+
const response = await github.rest.repos.createRelease({
|
| 381 |
+
draft: false,
|
| 382 |
+
generate_release_notes: true,
|
| 383 |
+
target_commitish: commitHash,
|
| 384 |
+
name: process.env.RELEASE_TAG,
|
| 385 |
+
owner: context.repo.owner,
|
| 386 |
+
prerelease: false,
|
| 387 |
+
repo: context.repo.repo,
|
| 388 |
+
tag_name: process.env.RELEASE_TAG,
|
| 389 |
+
});
|
| 390 |
+
|
| 391 |
+
core.exportVariable('RELEASE_ID', response.data.id);
|
| 392 |
+
core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url);
|
| 393 |
+
} catch (error) {
|
| 394 |
+
core.setFailed(error.message);
|
| 395 |
+
}
|
| 396 |
+
- name: Fetch Release Notes
|
| 397 |
+
id: release-notes
|
| 398 |
+
uses: actions/github-script@v6
|
| 399 |
+
with:
|
| 400 |
+
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
| 401 |
+
script: |
|
| 402 |
+
try {
|
| 403 |
+
const response = await github.rest.repos.getRelease({
|
| 404 |
+
owner: context.repo.owner,
|
| 405 |
+
repo: context.repo.repo,
|
| 406 |
+
release_id: process.env.RELEASE_ID,
|
| 407 |
+
});
|
| 408 |
+
const formattedBody = JSON.stringify(response.data.body).slice(1, -1);
|
| 409 |
+
return formattedBody;
|
| 410 |
+
} catch (error) {
|
| 411 |
+
core.setFailed(error.message);
|
| 412 |
+
}
|
| 413 |
+
env:
|
| 414 |
+
RELEASE_ID: ${{ env.RELEASE_ID }}
|
| 415 |
+
- name: Github Releases To Discord
|
| 416 |
+
env:
|
| 417 |
+
WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }}
|
| 418 |
+
REALEASE_TAG: ${{ env.RELEASE_TAG }}
|
| 419 |
+
RELEASE_NOTES: ${{ steps.release-notes.outputs.result }}
|
| 420 |
+
run: |
|
| 421 |
+
curl -H "Content-Type: application/json" -X POST -d '{
|
| 422 |
+
"content": "New LiteLLM release '"${RELEASE_TAG}"'",
|
| 423 |
+
"username": "Release Changelog",
|
| 424 |
+
"avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png",
|
| 425 |
+
"embeds": [
|
| 426 |
+
{
|
| 427 |
+
"title": "Changelog for LiteLLM '"${RELEASE_TAG}"'",
|
| 428 |
+
"description": "'"${RELEASE_NOTES}"'",
|
| 429 |
+
"color": 2105893
|
| 430 |
+
}
|
| 431 |
+
]
|
| 432 |
+
}' $WEBHOOK_URL
|
| 433 |
+
|
.github/workflows/ghcr_helm_deploy.yml
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
|
| 2 |
+
name: Build, Publish LiteLLM Helm Chart. New Release
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch:
|
| 5 |
+
inputs:
|
| 6 |
+
chartVersion:
|
| 7 |
+
description: "Update the helm chart's version to this"
|
| 8 |
+
|
| 9 |
+
# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
|
| 10 |
+
env:
|
| 11 |
+
REGISTRY: ghcr.io
|
| 12 |
+
IMAGE_NAME: ${{ github.repository }}
|
| 13 |
+
REPO_OWNER: ${{github.repository_owner}}
|
| 14 |
+
|
| 15 |
+
# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
|
| 16 |
+
jobs:
|
| 17 |
+
build-and-push-helm-chart:
|
| 18 |
+
runs-on: ubuntu-latest
|
| 19 |
+
steps:
|
| 20 |
+
- name: Checkout repository
|
| 21 |
+
uses: actions/checkout@v4
|
| 22 |
+
|
| 23 |
+
- name: Log in to the Container registry
|
| 24 |
+
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
|
| 25 |
+
with:
|
| 26 |
+
registry: ${{ env.REGISTRY }}
|
| 27 |
+
username: ${{ github.actor }}
|
| 28 |
+
password: ${{ secrets.GITHUB_TOKEN }}
|
| 29 |
+
|
| 30 |
+
- name: lowercase github.repository_owner
|
| 31 |
+
run: |
|
| 32 |
+
echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
|
| 33 |
+
|
| 34 |
+
- name: Get LiteLLM Latest Tag
|
| 35 |
+
id: current_app_tag
|
| 36 |
+
uses: WyriHaximus/[email protected]
|
| 37 |
+
|
| 38 |
+
- name: Get last published chart version
|
| 39 |
+
id: current_version
|
| 40 |
+
shell: bash
|
| 41 |
+
run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
|
| 42 |
+
env:
|
| 43 |
+
HELM_EXPERIMENTAL_OCI: '1'
|
| 44 |
+
|
| 45 |
+
# Automatically update the helm chart version one "patch" level
|
| 46 |
+
- name: Bump release version
|
| 47 |
+
id: bump_version
|
| 48 |
+
uses: christian-draeger/[email protected]
|
| 49 |
+
with:
|
| 50 |
+
current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
|
| 51 |
+
version-fragment: 'bug'
|
| 52 |
+
|
| 53 |
+
- name: Lint helm chart
|
| 54 |
+
run: helm lint deploy/charts/litellm-helm
|
| 55 |
+
|
| 56 |
+
- uses: ./.github/actions/helm-oci-chart-releaser
|
| 57 |
+
with:
|
| 58 |
+
name: litellm-helm
|
| 59 |
+
repository: ${{ env.REPO_OWNER }}
|
| 60 |
+
tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
|
| 61 |
+
app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
|
| 62 |
+
path: deploy/charts/litellm-helm
|
| 63 |
+
registry: ${{ env.REGISTRY }}
|
| 64 |
+
registry_username: ${{ github.actor }}
|
| 65 |
+
registry_password: ${{ secrets.GITHUB_TOKEN }}
|
| 66 |
+
update_dependencies: true
|
| 67 |
+
|
.github/workflows/helm_unit_test.yml
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Helm unit test
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
pull_request:
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- main
|
| 8 |
+
|
| 9 |
+
jobs:
|
| 10 |
+
unit-test:
|
| 11 |
+
runs-on: ubuntu-latest
|
| 12 |
+
steps:
|
| 13 |
+
- name: Checkout
|
| 14 |
+
uses: actions/checkout@v2
|
| 15 |
+
|
| 16 |
+
- name: Set up Helm 3.11.1
|
| 17 |
+
uses: azure/setup-helm@v1
|
| 18 |
+
with:
|
| 19 |
+
version: '3.11.1'
|
| 20 |
+
|
| 21 |
+
- name: Install Helm Unit Test Plugin
|
| 22 |
+
run: |
|
| 23 |
+
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
|
| 24 |
+
|
| 25 |
+
- name: Run unit tests
|
| 26 |
+
run:
|
| 27 |
+
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
|
.github/workflows/interpret_load_test.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import csv
|
| 2 |
+
import os
|
| 3 |
+
from github import Github
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def interpret_results(csv_file):
|
| 7 |
+
with open(csv_file, newline="") as csvfile:
|
| 8 |
+
csvreader = csv.DictReader(csvfile)
|
| 9 |
+
rows = list(csvreader)
|
| 10 |
+
"""
|
| 11 |
+
in this csv reader
|
| 12 |
+
- Create 1 new column "Status"
|
| 13 |
+
- if a row has a median response time < 300 and an average response time < 300, Status = "Passed ✅"
|
| 14 |
+
- if a row has a median response time >= 300 or an average response time >= 300, Status = "Failed ❌"
|
| 15 |
+
- Order the table in this order Name, Status, Median Response Time, Average Response Time, Requests/s,Failures/s, Min Response Time, Max Response Time, all other columns
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
# Add a new column "Status"
|
| 19 |
+
for row in rows:
|
| 20 |
+
median_response_time = float(
|
| 21 |
+
row["Median Response Time"].strip().rstrip("ms")
|
| 22 |
+
)
|
| 23 |
+
average_response_time = float(
|
| 24 |
+
row["Average Response Time"].strip().rstrip("s")
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
request_count = int(row["Request Count"])
|
| 28 |
+
failure_count = int(row["Failure Count"])
|
| 29 |
+
|
| 30 |
+
failure_percent = round((failure_count / request_count) * 100, 2)
|
| 31 |
+
|
| 32 |
+
# Determine status based on conditions
|
| 33 |
+
if (
|
| 34 |
+
median_response_time < 300
|
| 35 |
+
and average_response_time < 300
|
| 36 |
+
and failure_percent < 5
|
| 37 |
+
):
|
| 38 |
+
row["Status"] = "Passed ✅"
|
| 39 |
+
else:
|
| 40 |
+
row["Status"] = "Failed ❌"
|
| 41 |
+
|
| 42 |
+
# Construct Markdown table header
|
| 43 |
+
markdown_table = "| Name | Status | Median Response Time (ms) | Average Response Time (ms) | Requests/s | Failures/s | Request Count | Failure Count | Min Response Time (ms) | Max Response Time (ms) |"
|
| 44 |
+
markdown_table += (
|
| 45 |
+
"\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
# Construct Markdown table rows
|
| 49 |
+
for row in rows:
|
| 50 |
+
markdown_table += f"\n| {row['Name']} | {row['Status']} | {row['Median Response Time']} | {row['Average Response Time']} | {row['Requests/s']} | {row['Failures/s']} | {row['Request Count']} | {row['Failure Count']} | {row['Min Response Time']} | {row['Max Response Time']} |"
|
| 51 |
+
print("markdown table: ", markdown_table)
|
| 52 |
+
return markdown_table
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _get_docker_run_command_stable_release(release_version):
|
| 56 |
+
return f"""
|
| 57 |
+
\n\n
|
| 58 |
+
## Docker Run LiteLLM Proxy
|
| 59 |
+
|
| 60 |
+
```
|
| 61 |
+
docker run \\
|
| 62 |
+
-e STORE_MODEL_IN_DB=True \\
|
| 63 |
+
-p 4000:4000 \\
|
| 64 |
+
ghcr.io/berriai/litellm:litellm_stable_release_branch-{release_version}
|
| 65 |
+
```
|
| 66 |
+
"""
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def _get_docker_run_command(release_version):
|
| 70 |
+
return f"""
|
| 71 |
+
\n\n
|
| 72 |
+
## Docker Run LiteLLM Proxy
|
| 73 |
+
|
| 74 |
+
```
|
| 75 |
+
docker run \\
|
| 76 |
+
-e STORE_MODEL_IN_DB=True \\
|
| 77 |
+
-p 4000:4000 \\
|
| 78 |
+
ghcr.io/berriai/litellm:main-{release_version}
|
| 79 |
+
```
|
| 80 |
+
"""
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def get_docker_run_command(release_version):
|
| 84 |
+
if "stable" in release_version:
|
| 85 |
+
return _get_docker_run_command_stable_release(release_version)
|
| 86 |
+
else:
|
| 87 |
+
return _get_docker_run_command(release_version)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
if __name__ == "__main__":
|
| 91 |
+
csv_file = "load_test_stats.csv" # Change this to the path of your CSV file
|
| 92 |
+
markdown_table = interpret_results(csv_file)
|
| 93 |
+
|
| 94 |
+
# Update release body with interpreted results
|
| 95 |
+
github_token = os.getenv("GITHUB_TOKEN")
|
| 96 |
+
g = Github(github_token)
|
| 97 |
+
repo = g.get_repo(
|
| 98 |
+
"BerriAI/litellm"
|
| 99 |
+
) # Replace with your repository's username and name
|
| 100 |
+
latest_release = repo.get_latest_release()
|
| 101 |
+
print("got latest release: ", latest_release)
|
| 102 |
+
print(latest_release.title)
|
| 103 |
+
print(latest_release.tag_name)
|
| 104 |
+
|
| 105 |
+
release_version = latest_release.title
|
| 106 |
+
|
| 107 |
+
print("latest release body: ", latest_release.body)
|
| 108 |
+
print("markdown table: ", markdown_table)
|
| 109 |
+
|
| 110 |
+
# check if "Load Test LiteLLM Proxy Results" exists
|
| 111 |
+
existing_release_body = latest_release.body
|
| 112 |
+
if "Load Test LiteLLM Proxy Results" in latest_release.body:
|
| 113 |
+
# find the "Load Test LiteLLM Proxy Results" section and delete it
|
| 114 |
+
start_index = latest_release.body.find("Load Test LiteLLM Proxy Results")
|
| 115 |
+
existing_release_body = latest_release.body[:start_index]
|
| 116 |
+
|
| 117 |
+
docker_run_command = get_docker_run_command(release_version)
|
| 118 |
+
print("docker run command: ", docker_run_command)
|
| 119 |
+
|
| 120 |
+
new_release_body = (
|
| 121 |
+
existing_release_body
|
| 122 |
+
+ docker_run_command
|
| 123 |
+
+ "\n\n"
|
| 124 |
+
+ "### Don't want to maintain your internal proxy? get in touch 🎉"
|
| 125 |
+
+ "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
|
| 126 |
+
+ "\n\n"
|
| 127 |
+
+ "## Load Test LiteLLM Proxy Results"
|
| 128 |
+
+ "\n\n"
|
| 129 |
+
+ markdown_table
|
| 130 |
+
)
|
| 131 |
+
print("new release body: ", new_release_body)
|
| 132 |
+
try:
|
| 133 |
+
latest_release.update_release(
|
| 134 |
+
name=latest_release.tag_name,
|
| 135 |
+
message=new_release_body,
|
| 136 |
+
)
|
| 137 |
+
except Exception as e:
|
| 138 |
+
print(e)
|
.github/workflows/label-mlops.yml
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Label ML Ops Team Issues
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
issues:
|
| 5 |
+
types:
|
| 6 |
+
- opened
|
| 7 |
+
|
| 8 |
+
jobs:
|
| 9 |
+
add-mlops-label:
|
| 10 |
+
runs-on: ubuntu-latest
|
| 11 |
+
steps:
|
| 12 |
+
- name: Check if ML Ops Team is selected
|
| 13 |
+
uses: actions-ecosystem/action-add-labels@v1
|
| 14 |
+
if: contains(github.event.issue.body, '### Are you a ML Ops Team?') && contains(github.event.issue.body, 'Yes')
|
| 15 |
+
with:
|
| 16 |
+
github_token: ${{ secrets.GITHUB_TOKEN }}
|
| 17 |
+
labels: "mlops user request"
|
.github/workflows/load_test.yml
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Test Locust Load Test
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_run:
|
| 5 |
+
workflows: ["Build, Publish LiteLLM Docker Image. New Release"]
|
| 6 |
+
types:
|
| 7 |
+
- completed
|
| 8 |
+
workflow_dispatch:
|
| 9 |
+
|
| 10 |
+
jobs:
|
| 11 |
+
build:
|
| 12 |
+
runs-on: ubuntu-latest
|
| 13 |
+
steps:
|
| 14 |
+
- name: Checkout
|
| 15 |
+
uses: actions/checkout@v1
|
| 16 |
+
- name: Setup Python
|
| 17 |
+
uses: actions/setup-python@v2
|
| 18 |
+
with:
|
| 19 |
+
python-version: '3.x'
|
| 20 |
+
|
| 21 |
+
- name: Install dependencies
|
| 22 |
+
run: |
|
| 23 |
+
python -m pip install --upgrade pip
|
| 24 |
+
pip install PyGithub
|
| 25 |
+
- name: re-deploy proxy
|
| 26 |
+
run: |
|
| 27 |
+
echo "Current working directory: $PWD"
|
| 28 |
+
ls
|
| 29 |
+
python ".github/workflows/redeploy_proxy.py"
|
| 30 |
+
env:
|
| 31 |
+
LOAD_TEST_REDEPLOY_URL1: ${{ secrets.LOAD_TEST_REDEPLOY_URL1 }}
|
| 32 |
+
LOAD_TEST_REDEPLOY_URL2: ${{ secrets.LOAD_TEST_REDEPLOY_URL2 }}
|
| 33 |
+
working-directory: ${{ github.workspace }}
|
| 34 |
+
- name: Run Load Test
|
| 35 |
+
id: locust_run
|
| 36 |
+
uses: BerriAI/locust-github-action@master
|
| 37 |
+
with:
|
| 38 |
+
LOCUSTFILE: ".github/workflows/locustfile.py"
|
| 39 |
+
URL: "https://post-release-load-test-proxy.onrender.com/"
|
| 40 |
+
USERS: "20"
|
| 41 |
+
RATE: "20"
|
| 42 |
+
RUNTIME: "300s"
|
| 43 |
+
- name: Process Load Test Stats
|
| 44 |
+
run: |
|
| 45 |
+
echo "Current working directory: $PWD"
|
| 46 |
+
ls
|
| 47 |
+
python ".github/workflows/interpret_load_test.py"
|
| 48 |
+
env:
|
| 49 |
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
| 50 |
+
working-directory: ${{ github.workspace }}
|
| 51 |
+
- name: Upload CSV as Asset to Latest Release
|
| 52 |
+
uses: xresloader/upload-to-github-release@v1
|
| 53 |
+
env:
|
| 54 |
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
| 55 |
+
with:
|
| 56 |
+
file: "load_test_stats.csv;load_test.html"
|
| 57 |
+
update_latest_release: true
|
| 58 |
+
tag_name: "load-test"
|
| 59 |
+
overwrite: true
|
.github/workflows/locustfile.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from locust import HttpUser, task, between
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class MyUser(HttpUser):
|
| 5 |
+
wait_time = between(1, 5)
|
| 6 |
+
|
| 7 |
+
@task
|
| 8 |
+
def chat_completion(self):
|
| 9 |
+
headers = {
|
| 10 |
+
"Content-Type": "application/json",
|
| 11 |
+
"Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg",
|
| 12 |
+
# Include any additional headers you may need for authentication, etc.
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
# Customize the payload with "model" and "messages" keys
|
| 16 |
+
payload = {
|
| 17 |
+
"model": "fake-openai-endpoint",
|
| 18 |
+
"messages": [
|
| 19 |
+
{"role": "system", "content": "You are a chat bot."},
|
| 20 |
+
{"role": "user", "content": "Hello, how are you?"},
|
| 21 |
+
],
|
| 22 |
+
# Add more data as necessary
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
# Make a POST request to the "chat/completions" endpoint
|
| 26 |
+
response = self.client.post("chat/completions", json=payload, headers=headers)
|
| 27 |
+
|
| 28 |
+
# Print or log the response if needed
|
.github/workflows/main.yml
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Publish Dev Release to PyPI
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch:
|
| 5 |
+
|
| 6 |
+
jobs:
|
| 7 |
+
publish-dev-release:
|
| 8 |
+
runs-on: ubuntu-latest
|
| 9 |
+
|
| 10 |
+
steps:
|
| 11 |
+
- name: Checkout code
|
| 12 |
+
uses: actions/checkout@v2
|
| 13 |
+
|
| 14 |
+
- name: Set up Python
|
| 15 |
+
uses: actions/setup-python@v2
|
| 16 |
+
with:
|
| 17 |
+
python-version: 3.8 # Adjust the Python version as needed
|
| 18 |
+
|
| 19 |
+
- name: Install dependencies
|
| 20 |
+
run: pip install toml twine
|
| 21 |
+
|
| 22 |
+
- name: Read version from pyproject.toml
|
| 23 |
+
id: read-version
|
| 24 |
+
run: |
|
| 25 |
+
version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])')
|
| 26 |
+
printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
|
| 27 |
+
|
| 28 |
+
- name: Check if version exists on PyPI
|
| 29 |
+
id: check-version
|
| 30 |
+
run: |
|
| 31 |
+
set -e
|
| 32 |
+
if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then
|
| 33 |
+
echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish."
|
| 34 |
+
|
.github/workflows/publish-migrations.yml
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Publish Prisma Migrations
|
| 2 |
+
|
| 3 |
+
permissions:
|
| 4 |
+
contents: write
|
| 5 |
+
pull-requests: write
|
| 6 |
+
|
| 7 |
+
on:
|
| 8 |
+
push:
|
| 9 |
+
paths:
|
| 10 |
+
- 'schema.prisma' # Check root schema.prisma
|
| 11 |
+
branches:
|
| 12 |
+
- main
|
| 13 |
+
|
| 14 |
+
jobs:
|
| 15 |
+
publish-migrations:
|
| 16 |
+
runs-on: ubuntu-latest
|
| 17 |
+
services:
|
| 18 |
+
postgres:
|
| 19 |
+
image: postgres:14
|
| 20 |
+
env:
|
| 21 |
+
POSTGRES_DB: temp_db
|
| 22 |
+
POSTGRES_USER: postgres
|
| 23 |
+
POSTGRES_PASSWORD: postgres
|
| 24 |
+
ports:
|
| 25 |
+
- 5432:5432
|
| 26 |
+
options: >-
|
| 27 |
+
--health-cmd pg_isready
|
| 28 |
+
--health-interval 10s
|
| 29 |
+
--health-timeout 5s
|
| 30 |
+
--health-retries 5
|
| 31 |
+
|
| 32 |
+
# Add shadow database service
|
| 33 |
+
postgres_shadow:
|
| 34 |
+
image: postgres:14
|
| 35 |
+
env:
|
| 36 |
+
POSTGRES_DB: shadow_db
|
| 37 |
+
POSTGRES_USER: postgres
|
| 38 |
+
POSTGRES_PASSWORD: postgres
|
| 39 |
+
ports:
|
| 40 |
+
- 5433:5432
|
| 41 |
+
options: >-
|
| 42 |
+
--health-cmd pg_isready
|
| 43 |
+
--health-interval 10s
|
| 44 |
+
--health-timeout 5s
|
| 45 |
+
--health-retries 5
|
| 46 |
+
|
| 47 |
+
steps:
|
| 48 |
+
- uses: actions/checkout@v3
|
| 49 |
+
|
| 50 |
+
- name: Set up Python
|
| 51 |
+
uses: actions/setup-python@v4
|
| 52 |
+
with:
|
| 53 |
+
python-version: '3.x'
|
| 54 |
+
|
| 55 |
+
- name: Install Dependencies
|
| 56 |
+
run: |
|
| 57 |
+
pip install prisma
|
| 58 |
+
pip install python-dotenv
|
| 59 |
+
|
| 60 |
+
- name: Generate Initial Migration if None Exists
|
| 61 |
+
env:
|
| 62 |
+
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
|
| 63 |
+
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
|
| 64 |
+
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
|
| 65 |
+
run: |
|
| 66 |
+
mkdir -p deploy/migrations
|
| 67 |
+
echo 'provider = "postgresql"' > deploy/migrations/migration_lock.toml
|
| 68 |
+
|
| 69 |
+
if [ -z "$(ls -A deploy/migrations/2* 2>/dev/null)" ]; then
|
| 70 |
+
echo "No existing migrations found, creating baseline..."
|
| 71 |
+
VERSION=$(date +%Y%m%d%H%M%S)
|
| 72 |
+
mkdir -p deploy/migrations/${VERSION}_initial
|
| 73 |
+
|
| 74 |
+
echo "Generating initial migration..."
|
| 75 |
+
# Save raw output for debugging
|
| 76 |
+
prisma migrate diff \
|
| 77 |
+
--from-empty \
|
| 78 |
+
--to-schema-datamodel schema.prisma \
|
| 79 |
+
--shadow-database-url "${SHADOW_DATABASE_URL}" \
|
| 80 |
+
--script > deploy/migrations/${VERSION}_initial/raw_migration.sql
|
| 81 |
+
|
| 82 |
+
echo "Raw migration file content:"
|
| 83 |
+
cat deploy/migrations/${VERSION}_initial/raw_migration.sql
|
| 84 |
+
|
| 85 |
+
echo "Cleaning migration file..."
|
| 86 |
+
# Clean the file
|
| 87 |
+
sed '/^Installing/d' deploy/migrations/${VERSION}_initial/raw_migration.sql > deploy/migrations/${VERSION}_initial/migration.sql
|
| 88 |
+
|
| 89 |
+
# Verify the migration file
|
| 90 |
+
if [ ! -s deploy/migrations/${VERSION}_initial/migration.sql ]; then
|
| 91 |
+
echo "ERROR: Migration file is empty after cleaning"
|
| 92 |
+
echo "Original content was:"
|
| 93 |
+
cat deploy/migrations/${VERSION}_initial/raw_migration.sql
|
| 94 |
+
exit 1
|
| 95 |
+
fi
|
| 96 |
+
|
| 97 |
+
echo "Final migration file content:"
|
| 98 |
+
cat deploy/migrations/${VERSION}_initial/migration.sql
|
| 99 |
+
|
| 100 |
+
# Verify it starts with SQL
|
| 101 |
+
if ! head -n 1 deploy/migrations/${VERSION}_initial/migration.sql | grep -q "^--\|^CREATE\|^ALTER"; then
|
| 102 |
+
echo "ERROR: Migration file does not start with SQL command or comment"
|
| 103 |
+
echo "First line is:"
|
| 104 |
+
head -n 1 deploy/migrations/${VERSION}_initial/migration.sql
|
| 105 |
+
echo "Full content is:"
|
| 106 |
+
cat deploy/migrations/${VERSION}_initial/migration.sql
|
| 107 |
+
exit 1
|
| 108 |
+
fi
|
| 109 |
+
|
| 110 |
+
echo "Initial migration generated at $(date -u)" > deploy/migrations/${VERSION}_initial/README.md
|
| 111 |
+
fi
|
| 112 |
+
|
| 113 |
+
- name: Compare and Generate Migration
|
| 114 |
+
if: success()
|
| 115 |
+
env:
|
| 116 |
+
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
|
| 117 |
+
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
|
| 118 |
+
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
|
| 119 |
+
run: |
|
| 120 |
+
# Create temporary migration workspace
|
| 121 |
+
mkdir -p temp_migrations
|
| 122 |
+
|
| 123 |
+
# Copy existing migrations (will not fail if directory is empty)
|
| 124 |
+
cp -r deploy/migrations/* temp_migrations/ 2>/dev/null || true
|
| 125 |
+
|
| 126 |
+
VERSION=$(date +%Y%m%d%H%M%S)
|
| 127 |
+
|
| 128 |
+
# Generate diff against existing migrations or empty state
|
| 129 |
+
prisma migrate diff \
|
| 130 |
+
--from-migrations temp_migrations \
|
| 131 |
+
--to-schema-datamodel schema.prisma \
|
| 132 |
+
--shadow-database-url "${SHADOW_DATABASE_URL}" \
|
| 133 |
+
--script > temp_migrations/migration_${VERSION}.sql
|
| 134 |
+
|
| 135 |
+
# Check if there are actual changes
|
| 136 |
+
if [ -s temp_migrations/migration_${VERSION}.sql ]; then
|
| 137 |
+
echo "Changes detected, creating new migration"
|
| 138 |
+
mkdir -p deploy/migrations/${VERSION}_schema_update
|
| 139 |
+
mv temp_migrations/migration_${VERSION}.sql deploy/migrations/${VERSION}_schema_update/migration.sql
|
| 140 |
+
echo "Migration generated at $(date -u)" > deploy/migrations/${VERSION}_schema_update/README.md
|
| 141 |
+
else
|
| 142 |
+
echo "No schema changes detected"
|
| 143 |
+
exit 0
|
| 144 |
+
fi
|
| 145 |
+
|
| 146 |
+
- name: Verify Migration
|
| 147 |
+
if: success()
|
| 148 |
+
env:
|
| 149 |
+
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
|
| 150 |
+
DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
|
| 151 |
+
SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
|
| 152 |
+
run: |
|
| 153 |
+
# Create test database
|
| 154 |
+
psql "${SHADOW_DATABASE_URL}" -c 'CREATE DATABASE migration_test;'
|
| 155 |
+
|
| 156 |
+
# Apply all migrations in order to verify
|
| 157 |
+
for migration in deploy/migrations/*/migration.sql; do
|
| 158 |
+
echo "Applying migration: $migration"
|
| 159 |
+
psql "${SHADOW_DATABASE_URL}" -f $migration
|
| 160 |
+
done
|
| 161 |
+
|
| 162 |
+
# Add this step before create-pull-request to debug permissions
|
| 163 |
+
- name: Check Token Permissions
|
| 164 |
+
run: |
|
| 165 |
+
echo "Checking token permissions..."
|
| 166 |
+
curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
| 167 |
+
-H "Accept: application/vnd.github.v3+json" \
|
| 168 |
+
https://api.github.com/repos/BerriAI/litellm/collaborators
|
| 169 |
+
|
| 170 |
+
echo "\nChecking if token can create PRs..."
|
| 171 |
+
curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
| 172 |
+
-H "Accept: application/vnd.github.v3+json" \
|
| 173 |
+
https://api.github.com/repos/BerriAI/litellm
|
| 174 |
+
|
| 175 |
+
# Add this debug step before git push
|
| 176 |
+
- name: Debug Changed Files
|
| 177 |
+
run: |
|
| 178 |
+
echo "Files staged for commit:"
|
| 179 |
+
git diff --name-status --staged
|
| 180 |
+
|
| 181 |
+
echo "\nAll changed files:"
|
| 182 |
+
git status
|
| 183 |
+
|
| 184 |
+
- name: Create Pull Request
|
| 185 |
+
if: success()
|
| 186 |
+
uses: peter-evans/create-pull-request@v5
|
| 187 |
+
with:
|
| 188 |
+
token: ${{ secrets.GITHUB_TOKEN }}
|
| 189 |
+
commit-message: "chore: update prisma migrations"
|
| 190 |
+
title: "Update Prisma Migrations"
|
| 191 |
+
body: |
|
| 192 |
+
Auto-generated migration based on schema.prisma changes.
|
| 193 |
+
|
| 194 |
+
Generated files:
|
| 195 |
+
- deploy/migrations/${VERSION}_schema_update/migration.sql
|
| 196 |
+
- deploy/migrations/${VERSION}_schema_update/README.md
|
| 197 |
+
branch: feat/prisma-migration-${{ env.VERSION }}
|
| 198 |
+
base: main
|
| 199 |
+
delete-branch: true
|
| 200 |
+
|
| 201 |
+
- name: Generate and Save Migrations
|
| 202 |
+
run: |
|
| 203 |
+
# Only add migration files
|
| 204 |
+
git add deploy/migrations/
|
| 205 |
+
git status # Debug what's being committed
|
| 206 |
+
git commit -m "chore: update prisma migrations"
|
.github/workflows/read_pyproject_version.yml
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Read Version from pyproject.toml
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
branches:
|
| 6 |
+
- main # Change this to the default branch of your repository
|
| 7 |
+
|
| 8 |
+
jobs:
|
| 9 |
+
read-version:
|
| 10 |
+
runs-on: ubuntu-latest
|
| 11 |
+
|
| 12 |
+
steps:
|
| 13 |
+
- name: Checkout code
|
| 14 |
+
uses: actions/checkout@v2
|
| 15 |
+
|
| 16 |
+
- name: Set up Python
|
| 17 |
+
uses: actions/setup-python@v2
|
| 18 |
+
with:
|
| 19 |
+
python-version: 3.8 # Adjust the Python version as needed
|
| 20 |
+
|
| 21 |
+
- name: Install dependencies
|
| 22 |
+
run: pip install toml
|
| 23 |
+
|
| 24 |
+
- name: Read version from pyproject.toml
|
| 25 |
+
id: read-version
|
| 26 |
+
run: |
|
| 27 |
+
version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])')
|
| 28 |
+
printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
|
| 29 |
+
|
| 30 |
+
- name: Display version
|
| 31 |
+
run: echo "Current version is $LITELLM_VERSION"
|
.github/workflows/redeploy_proxy.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
|
| 3 |
+
redeploy_proxy.py
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import os
|
| 7 |
+
import requests
|
| 8 |
+
import time
|
| 9 |
+
|
| 10 |
+
# send a get request to this endpoint
|
| 11 |
+
deploy_hook1 = os.getenv("LOAD_TEST_REDEPLOY_URL1")
|
| 12 |
+
response = requests.get(deploy_hook1, timeout=20)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
deploy_hook2 = os.getenv("LOAD_TEST_REDEPLOY_URL2")
|
| 16 |
+
response = requests.get(deploy_hook2, timeout=20)
|
| 17 |
+
|
| 18 |
+
print("SENT GET REQUESTS to re-deploy proxy")
|
| 19 |
+
print("sleeeping.... for 60s")
|
| 20 |
+
time.sleep(60)
|
.github/workflows/reset_stable.yml
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Reset litellm_stable branch
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
release:
|
| 5 |
+
types: [published, created]
|
| 6 |
+
jobs:
|
| 7 |
+
update-stable-branch:
|
| 8 |
+
if: ${{ startsWith(github.event.release.tag_name, 'v') && !endsWith(github.event.release.tag_name, '-stable') }}
|
| 9 |
+
runs-on: ubuntu-latest
|
| 10 |
+
|
| 11 |
+
steps:
|
| 12 |
+
- name: Checkout repository
|
| 13 |
+
uses: actions/checkout@v3
|
| 14 |
+
|
| 15 |
+
- name: Reset litellm_stable_release_branch branch to the release commit
|
| 16 |
+
env:
|
| 17 |
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
| 18 |
+
run: |
|
| 19 |
+
# Configure Git user
|
| 20 |
+
git config user.name "github-actions[bot]"
|
| 21 |
+
git config user.email "github-actions[bot]@users.noreply.github.com"
|
| 22 |
+
|
| 23 |
+
# Fetch all branches and tags
|
| 24 |
+
git fetch --all
|
| 25 |
+
|
| 26 |
+
# Check if the litellm_stable_release_branch branch exists
|
| 27 |
+
if git show-ref --verify --quiet refs/remotes/origin/litellm_stable_release_branch; then
|
| 28 |
+
echo "litellm_stable_release_branch branch exists."
|
| 29 |
+
git checkout litellm_stable_release_branch
|
| 30 |
+
else
|
| 31 |
+
echo "litellm_stable_release_branch branch does not exist. Creating it."
|
| 32 |
+
git checkout -b litellm_stable_release_branch
|
| 33 |
+
fi
|
| 34 |
+
|
| 35 |
+
# Reset litellm_stable_release_branch branch to the release commit
|
| 36 |
+
git reset --hard $GITHUB_SHA
|
| 37 |
+
|
| 38 |
+
# Push the updated litellm_stable_release_branch branch
|
| 39 |
+
git push origin litellm_stable_release_branch --force
|
.github/workflows/results_stats.csv
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Date,"Ben
|
| 2 |
+
Ashley",Tom Brooks,Jimmy Cooney,"Sue
|
| 3 |
+
Daniels",Berlinda Fong,Terry Jones,Angelina Little,Linda Smith
|
| 4 |
+
10/1,FALSE,TRUE,TRUE,TRUE,TRUE,TRUE,FALSE,TRUE
|
| 5 |
+
10/2,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 6 |
+
10/3,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 7 |
+
10/4,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 8 |
+
10/5,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 9 |
+
10/6,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 10 |
+
10/7,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 11 |
+
10/8,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 12 |
+
10/9,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 13 |
+
10/10,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 14 |
+
10/11,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 15 |
+
10/12,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 16 |
+
10/13,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 17 |
+
10/14,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 18 |
+
10/15,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 19 |
+
10/16,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 20 |
+
10/17,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 21 |
+
10/18,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 22 |
+
10/19,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 23 |
+
10/20,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 24 |
+
10/21,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 25 |
+
10/22,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 26 |
+
10/23,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
|
| 27 |
+
Total,0,1,1,1,1,1,0,1
|
.github/workflows/stale.yml
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: "Stale Issue Management"
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
schedule:
|
| 5 |
+
- cron: '0 0 * * *' # Runs daily at midnight UTC
|
| 6 |
+
workflow_dispatch:
|
| 7 |
+
|
| 8 |
+
jobs:
|
| 9 |
+
stale:
|
| 10 |
+
runs-on: ubuntu-latest
|
| 11 |
+
steps:
|
| 12 |
+
- uses: actions/stale@v8
|
| 13 |
+
with:
|
| 14 |
+
repo-token: "${{ secrets.GITHUB_TOKEN }}"
|
| 15 |
+
stale-issue-message: "This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
|
| 16 |
+
stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
|
| 17 |
+
days-before-stale: 90 # Revert to 60 days
|
| 18 |
+
days-before-close: 7 # Revert to 7 days
|
| 19 |
+
stale-issue-label: "stale"
|
| 20 |
+
operations-per-run: 1000
|
.github/workflows/test-linting.yml
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: LiteLLM Linting
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
pull_request:
|
| 5 |
+
branches: [ main ]
|
| 6 |
+
|
| 7 |
+
jobs:
|
| 8 |
+
lint:
|
| 9 |
+
runs-on: ubuntu-latest
|
| 10 |
+
timeout-minutes: 5
|
| 11 |
+
|
| 12 |
+
steps:
|
| 13 |
+
- uses: actions/checkout@v4
|
| 14 |
+
|
| 15 |
+
- name: Set up Python
|
| 16 |
+
uses: actions/setup-python@v4
|
| 17 |
+
with:
|
| 18 |
+
python-version: '3.12'
|
| 19 |
+
|
| 20 |
+
- name: Install Poetry
|
| 21 |
+
uses: snok/install-poetry@v1
|
| 22 |
+
|
| 23 |
+
- name: Install dependencies
|
| 24 |
+
run: |
|
| 25 |
+
pip install openai==1.81.0
|
| 26 |
+
poetry install --with dev
|
| 27 |
+
pip install openai==1.81.0
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
- name: Run Black formatting
|
| 32 |
+
run: |
|
| 33 |
+
cd litellm
|
| 34 |
+
poetry run black .
|
| 35 |
+
cd ..
|
| 36 |
+
|
| 37 |
+
- name: Run Ruff linting
|
| 38 |
+
run: |
|
| 39 |
+
cd litellm
|
| 40 |
+
poetry run ruff check .
|
| 41 |
+
cd ..
|
| 42 |
+
|
| 43 |
+
- name: Run MyPy type checking
|
| 44 |
+
run: |
|
| 45 |
+
cd litellm
|
| 46 |
+
poetry run mypy . --ignore-missing-imports
|
| 47 |
+
cd ..
|
| 48 |
+
|
| 49 |
+
- name: Check for circular imports
|
| 50 |
+
run: |
|
| 51 |
+
cd litellm
|
| 52 |
+
poetry run python ../tests/documentation_tests/test_circular_imports.py
|
| 53 |
+
cd ..
|
| 54 |
+
|
| 55 |
+
- name: Check import safety
|
| 56 |
+
run: |
|
| 57 |
+
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
|
.github/workflows/test-litellm.yml
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: LiteLLM Mock Tests (folder - tests/test_litellm)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
pull_request:
|
| 5 |
+
branches: [ main ]
|
| 6 |
+
|
| 7 |
+
jobs:
|
| 8 |
+
test:
|
| 9 |
+
runs-on: ubuntu-latest
|
| 10 |
+
timeout-minutes: 15
|
| 11 |
+
|
| 12 |
+
steps:
|
| 13 |
+
- uses: actions/checkout@v4
|
| 14 |
+
|
| 15 |
+
- name: Thank You Message
|
| 16 |
+
run: |
|
| 17 |
+
echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
|
| 18 |
+
echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
|
| 19 |
+
|
| 20 |
+
- name: Set up Python
|
| 21 |
+
uses: actions/setup-python@v4
|
| 22 |
+
with:
|
| 23 |
+
python-version: '3.12'
|
| 24 |
+
|
| 25 |
+
- name: Install Poetry
|
| 26 |
+
uses: snok/install-poetry@v1
|
| 27 |
+
|
| 28 |
+
- name: Install dependencies
|
| 29 |
+
run: |
|
| 30 |
+
poetry install --with dev,proxy-dev --extras proxy
|
| 31 |
+
poetry run pip install "pytest-retry==1.6.3"
|
| 32 |
+
poetry run pip install pytest-xdist
|
| 33 |
+
- name: Setup litellm-enterprise as local package
|
| 34 |
+
run: |
|
| 35 |
+
cd enterprise
|
| 36 |
+
python -m pip install -e .
|
| 37 |
+
cd ..
|
| 38 |
+
- name: Run tests
|
| 39 |
+
run: |
|
| 40 |
+
poetry run pytest tests/test_litellm -x -vv -n 4
|
.github/workflows/update_release.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import requests
|
| 3 |
+
from datetime import datetime
|
| 4 |
+
|
| 5 |
+
# GitHub API endpoints
|
| 6 |
+
GITHUB_API_URL = "https://api.github.com"
|
| 7 |
+
REPO_OWNER = "BerriAI"
|
| 8 |
+
REPO_NAME = "litellm"
|
| 9 |
+
|
| 10 |
+
# GitHub personal access token (required for uploading release assets)
|
| 11 |
+
GITHUB_ACCESS_TOKEN = os.environ.get("GITHUB_ACCESS_TOKEN")
|
| 12 |
+
|
| 13 |
+
# Headers for GitHub API requests
|
| 14 |
+
headers = {
|
| 15 |
+
"Accept": "application/vnd.github+json",
|
| 16 |
+
"Authorization": f"Bearer {GITHUB_ACCESS_TOKEN}",
|
| 17 |
+
"X-GitHub-Api-Version": "2022-11-28",
|
| 18 |
+
}
|
| 19 |
+
|
| 20 |
+
# Get the latest release
|
| 21 |
+
releases_url = f"{GITHUB_API_URL}/repos/{REPO_OWNER}/{REPO_NAME}/releases/latest"
|
| 22 |
+
response = requests.get(releases_url, headers=headers)
|
| 23 |
+
latest_release = response.json()
|
| 24 |
+
print("Latest release:", latest_release)
|
| 25 |
+
|
| 26 |
+
# Upload an asset to the latest release
|
| 27 |
+
upload_url = latest_release["upload_url"].split("{?")[0]
|
| 28 |
+
asset_name = "results_stats.csv"
|
| 29 |
+
asset_path = os.path.join(os.getcwd(), asset_name)
|
| 30 |
+
print("upload_url:", upload_url)
|
| 31 |
+
|
| 32 |
+
with open(asset_path, "rb") as asset_file:
|
| 33 |
+
asset_data = asset_file.read()
|
| 34 |
+
|
| 35 |
+
upload_payload = {
|
| 36 |
+
"name": asset_name,
|
| 37 |
+
"label": "Load test results",
|
| 38 |
+
"created_at": datetime.utcnow().isoformat() + "Z",
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
upload_headers = headers.copy()
|
| 42 |
+
upload_headers["Content-Type"] = "application/octet-stream"
|
| 43 |
+
|
| 44 |
+
upload_response = requests.post(
|
| 45 |
+
upload_url,
|
| 46 |
+
headers=upload_headers,
|
| 47 |
+
data=asset_data,
|
| 48 |
+
params=upload_payload,
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
if upload_response.status_code == 201:
|
| 52 |
+
print(f"Asset '{asset_name}' uploaded successfully to the latest release.")
|
| 53 |
+
else:
|
| 54 |
+
print(f"Failed to upload asset. Response: {upload_response.text}")
|
.pre-commit-config.yaml
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
repos:
|
| 2 |
+
- repo: local
|
| 3 |
+
hooks:
|
| 4 |
+
- id: pyright
|
| 5 |
+
name: pyright
|
| 6 |
+
entry: pyright
|
| 7 |
+
language: system
|
| 8 |
+
types: [python]
|
| 9 |
+
files: ^(litellm/|litellm_proxy_extras/|enterprise/)
|
| 10 |
+
- id: isort
|
| 11 |
+
name: isort
|
| 12 |
+
entry: isort
|
| 13 |
+
language: system
|
| 14 |
+
types: [python]
|
| 15 |
+
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
|
| 16 |
+
exclude: ^litellm/__init__.py$
|
| 17 |
+
- id: black
|
| 18 |
+
name: black
|
| 19 |
+
entry: poetry run black
|
| 20 |
+
language: system
|
| 21 |
+
types: [python]
|
| 22 |
+
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
|
| 23 |
+
- repo: https://github.com/pycqa/flake8
|
| 24 |
+
rev: 7.0.0 # The version of flake8 to use
|
| 25 |
+
hooks:
|
| 26 |
+
- id: flake8
|
| 27 |
+
exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/
|
| 28 |
+
additional_dependencies: [flake8-print]
|
| 29 |
+
files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
|
| 30 |
+
- repo: https://github.com/python-poetry/poetry
|
| 31 |
+
rev: 1.8.0
|
| 32 |
+
hooks:
|
| 33 |
+
- id: poetry-check
|
| 34 |
+
files: ^(pyproject.toml|litellm-proxy-extras/pyproject.toml)$
|
| 35 |
+
- repo: local
|
| 36 |
+
hooks:
|
| 37 |
+
- id: check-files-match
|
| 38 |
+
name: Check if files match
|
| 39 |
+
entry: python3 ci_cd/check_files_match.py
|
| 40 |
+
language: system
|
AGENTS.md
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# INSTRUCTIONS FOR LITELLM
|
| 2 |
+
|
| 3 |
+
This document provides comprehensive instructions for AI agents working in the LiteLLM repository.
|
| 4 |
+
|
| 5 |
+
## OVERVIEW
|
| 6 |
+
|
| 7 |
+
LiteLLM is a unified interface for 100+ LLMs that:
|
| 8 |
+
- Translates inputs to provider-specific completion, embedding, and image generation endpoints
|
| 9 |
+
- Provides consistent OpenAI-format output across all providers
|
| 10 |
+
- Includes retry/fallback logic across multiple deployments (Router)
|
| 11 |
+
- Offers a proxy server (LLM Gateway) with budgets, rate limits, and authentication
|
| 12 |
+
- Supports advanced features like function calling, streaming, caching, and observability
|
| 13 |
+
|
| 14 |
+
## REPOSITORY STRUCTURE
|
| 15 |
+
|
| 16 |
+
### Core Components
|
| 17 |
+
- `litellm/` - Main library code
|
| 18 |
+
- `llms/` - Provider-specific implementations (OpenAI, Anthropic, Azure, etc.)
|
| 19 |
+
- `proxy/` - Proxy server implementation (LLM Gateway)
|
| 20 |
+
- `router_utils/` - Load balancing and fallback logic
|
| 21 |
+
- `types/` - Type definitions and schemas
|
| 22 |
+
- `integrations/` - Third-party integrations (observability, caching, etc.)
|
| 23 |
+
|
| 24 |
+
### Key Directories
|
| 25 |
+
- `tests/` - Comprehensive test suites
|
| 26 |
+
- `docs/my-website/` - Documentation website
|
| 27 |
+
- `ui/litellm-dashboard/` - Admin dashboard UI
|
| 28 |
+
- `enterprise/` - Enterprise-specific features
|
| 29 |
+
|
| 30 |
+
## DEVELOPMENT GUIDELINES
|
| 31 |
+
|
| 32 |
+
### MAKING CODE CHANGES
|
| 33 |
+
|
| 34 |
+
1. **Provider Implementations**: When adding/modifying LLM providers:
|
| 35 |
+
- Follow existing patterns in `litellm/llms/{provider}/`
|
| 36 |
+
- Implement proper transformation classes that inherit from `BaseConfig`
|
| 37 |
+
- Support both sync and async operations
|
| 38 |
+
- Handle streaming responses appropriately
|
| 39 |
+
- Include proper error handling with provider-specific exceptions
|
| 40 |
+
|
| 41 |
+
2. **Type Safety**:
|
| 42 |
+
- Use proper type hints throughout
|
| 43 |
+
- Update type definitions in `litellm/types/`
|
| 44 |
+
- Ensure compatibility with both Pydantic v1 and v2
|
| 45 |
+
|
| 46 |
+
3. **Testing**:
|
| 47 |
+
- Add tests in appropriate `tests/` subdirectories
|
| 48 |
+
- Include both unit tests and integration tests
|
| 49 |
+
- Test provider-specific functionality thoroughly
|
| 50 |
+
- Consider adding load tests for performance-critical changes
|
| 51 |
+
|
| 52 |
+
### IMPORTANT PATTERNS
|
| 53 |
+
|
| 54 |
+
1. **Function/Tool Calling**:
|
| 55 |
+
- LiteLLM standardizes tool calling across providers
|
| 56 |
+
- OpenAI format is the standard, with transformations for other providers
|
| 57 |
+
- See `litellm/llms/anthropic/chat/transformation.py` for complex tool handling
|
| 58 |
+
|
| 59 |
+
2. **Streaming**:
|
| 60 |
+
- All providers should support streaming where possible
|
| 61 |
+
- Use consistent chunk formatting across providers
|
| 62 |
+
- Handle both sync and async streaming
|
| 63 |
+
|
| 64 |
+
3. **Error Handling**:
|
| 65 |
+
- Use provider-specific exception classes
|
| 66 |
+
- Maintain consistent error formats across providers
|
| 67 |
+
- Include proper retry logic and fallback mechanisms
|
| 68 |
+
|
| 69 |
+
4. **Configuration**:
|
| 70 |
+
- Support both environment variables and programmatic configuration
|
| 71 |
+
- Use `BaseConfig` classes for provider configurations
|
| 72 |
+
- Allow dynamic parameter passing
|
| 73 |
+
|
| 74 |
+
## PROXY SERVER (LLM GATEWAY)
|
| 75 |
+
|
| 76 |
+
The proxy server is a critical component that provides:
|
| 77 |
+
- Authentication and authorization
|
| 78 |
+
- Rate limiting and budget management
|
| 79 |
+
- Load balancing across multiple models/deployments
|
| 80 |
+
- Observability and logging
|
| 81 |
+
- Admin dashboard UI
|
| 82 |
+
- Enterprise features
|
| 83 |
+
|
| 84 |
+
Key files:
|
| 85 |
+
- `litellm/proxy/proxy_server.py` - Main server implementation
|
| 86 |
+
- `litellm/proxy/auth/` - Authentication logic
|
| 87 |
+
- `litellm/proxy/management_endpoints/` - Admin API endpoints
|
| 88 |
+
|
| 89 |
+
## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
|
| 90 |
+
|
| 91 |
+
LiteLLM supports MCP for agent workflows:
|
| 92 |
+
- MCP server integration for tool calling
|
| 93 |
+
- Transformation between OpenAI and MCP tool formats
|
| 94 |
+
- Support for external MCP servers (Zapier, Jira, Linear, etc.)
|
| 95 |
+
- See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/`
|
| 96 |
+
|
| 97 |
+
## TESTING CONSIDERATIONS
|
| 98 |
+
|
| 99 |
+
1. **Provider Tests**: Test against real provider APIs when possible
|
| 100 |
+
2. **Proxy Tests**: Include authentication, rate limiting, and routing tests
|
| 101 |
+
3. **Performance Tests**: Load testing for high-throughput scenarios
|
| 102 |
+
4. **Integration Tests**: End-to-end workflows including tool calling
|
| 103 |
+
|
| 104 |
+
## DOCUMENTATION
|
| 105 |
+
|
| 106 |
+
- Keep documentation in sync with code changes
|
| 107 |
+
- Update provider documentation when adding new providers
|
| 108 |
+
- Include code examples for new features
|
| 109 |
+
- Update changelog and release notes
|
| 110 |
+
|
| 111 |
+
## SECURITY CONSIDERATIONS
|
| 112 |
+
|
| 113 |
+
- Handle API keys securely
|
| 114 |
+
- Validate all inputs, especially for proxy endpoints
|
| 115 |
+
- Consider rate limiting and abuse prevention
|
| 116 |
+
- Follow security best practices for authentication
|
| 117 |
+
|
| 118 |
+
## ENTERPRISE FEATURES
|
| 119 |
+
|
| 120 |
+
- Some features are enterprise-only
|
| 121 |
+
- Check `enterprise/` directory for enterprise-specific code
|
| 122 |
+
- Maintain compatibility between open-source and enterprise versions
|
| 123 |
+
|
| 124 |
+
## COMMON PITFALLS TO AVOID
|
| 125 |
+
|
| 126 |
+
1. **Breaking Changes**: LiteLLM has many users - avoid breaking existing APIs
|
| 127 |
+
2. **Provider Specifics**: Each provider has unique quirks - handle them properly
|
| 128 |
+
3. **Rate Limits**: Respect provider rate limits in tests
|
| 129 |
+
4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
|
| 130 |
+
5. **Dependencies**: Keep dependencies minimal and well-justified
|
| 131 |
+
|
| 132 |
+
## HELPFUL RESOURCES
|
| 133 |
+
|
| 134 |
+
- Main documentation: https://docs.litellm.ai/
|
| 135 |
+
- Provider-specific docs in `docs/my-website/docs/providers/`
|
| 136 |
+
- Admin UI for testing proxy features
|
| 137 |
+
|
| 138 |
+
## WHEN IN DOUBT
|
| 139 |
+
|
| 140 |
+
- Follow existing patterns in the codebase
|
| 141 |
+
- Check similar provider implementations
|
| 142 |
+
- Ensure comprehensive test coverage
|
| 143 |
+
- Update documentation appropriately
|
| 144 |
+
- Consider backward compatibility impact
|
CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Contributing to LiteLLM
|
| 2 |
+
|
| 3 |
+
Thank you for your interest in contributing to LiteLLM! We welcome contributions of all kinds - from bug fixes and documentation improvements to new features and integrations.
|
| 4 |
+
|
| 5 |
+
## **Checklist before submitting a PR**
|
| 6 |
+
|
| 7 |
+
Here are the core requirements for any PR submitted to LiteLLM:
|
| 8 |
+
|
| 9 |
+
- [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
|
| 10 |
+
- [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
|
| 11 |
+
- [ ] **Ensure your PR passes all checks**:
|
| 12 |
+
- [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
|
| 13 |
+
- [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
|
| 14 |
+
- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
|
| 15 |
+
|
| 16 |
+
## **Contributor License Agreement (CLA)**
|
| 17 |
+
|
| 18 |
+
Before contributing code to LiteLLM, you must sign our [Contributor License Agreement (CLA)](https://cla-assistant.io/BerriAI/litellm). This is a legal requirement for all contributions to be merged into the main repository.
|
| 19 |
+
|
| 20 |
+
**Important:** We strongly recommend reviewing and signing the CLA before starting work on your contribution to avoid any delays in the PR process.
|
| 21 |
+
|
| 22 |
+
## Quick Start
|
| 23 |
+
|
| 24 |
+
### 1. Setup Your Local Development Environment
|
| 25 |
+
|
| 26 |
+
```bash
|
| 27 |
+
# Clone the repository
|
| 28 |
+
git clone https://github.com/BerriAI/litellm.git
|
| 29 |
+
cd litellm
|
| 30 |
+
|
| 31 |
+
# Create a new branch for your feature
|
| 32 |
+
git checkout -b your-feature-branch
|
| 33 |
+
|
| 34 |
+
# Install development dependencies
|
| 35 |
+
make install-dev
|
| 36 |
+
|
| 37 |
+
# Verify your setup works
|
| 38 |
+
make help
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
That's it! Your local development environment is ready.
|
| 42 |
+
|
| 43 |
+
### 2. Development Workflow
|
| 44 |
+
|
| 45 |
+
Here's the recommended workflow for making changes:
|
| 46 |
+
|
| 47 |
+
```bash
|
| 48 |
+
# Make your changes to the code
|
| 49 |
+
# ...
|
| 50 |
+
|
| 51 |
+
# Format your code (auto-fixes formatting issues)
|
| 52 |
+
make format
|
| 53 |
+
|
| 54 |
+
# Run all linting checks (matches CI exactly)
|
| 55 |
+
make lint
|
| 56 |
+
|
| 57 |
+
# Run unit tests to ensure nothing is broken
|
| 58 |
+
make test-unit
|
| 59 |
+
|
| 60 |
+
# Commit your changes
|
| 61 |
+
git add .
|
| 62 |
+
git commit -m "Your descriptive commit message"
|
| 63 |
+
|
| 64 |
+
# Push and create a PR
|
| 65 |
+
git push origin your-feature-branch
|
| 66 |
+
```
|
| 67 |
+
|
| 68 |
+
## Adding Testing
|
| 69 |
+
|
| 70 |
+
**Adding at least 1 test is a hard requirement for all PRs.**
|
| 71 |
+
|
| 72 |
+
### Where to Add Tests
|
| 73 |
+
|
| 74 |
+
Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm).
|
| 75 |
+
|
| 76 |
+
- This directory mirrors the structure of the `litellm/` directory
|
| 77 |
+
- **Only add mocked tests** - no real LLM API calls in this directory
|
| 78 |
+
- For integration tests with real APIs, use the appropriate test directories
|
| 79 |
+
|
| 80 |
+
### File Naming Convention
|
| 81 |
+
|
| 82 |
+
The `tests/test_litellm/` directory follows the same structure as `litellm/`:
|
| 83 |
+
|
| 84 |
+
- `litellm/proxy/caching_routes.py` → `tests/test_litellm/proxy/test_caching_routes.py`
|
| 85 |
+
- `litellm/utils.py` → `tests/test_litellm/test_utils.py`
|
| 86 |
+
|
| 87 |
+
### Example Test
|
| 88 |
+
|
| 89 |
+
```python
|
| 90 |
+
import pytest
|
| 91 |
+
from litellm import completion
|
| 92 |
+
|
| 93 |
+
def test_your_feature():
|
| 94 |
+
"""Test your feature with a descriptive docstring."""
|
| 95 |
+
# Arrange
|
| 96 |
+
messages = [{"role": "user", "content": "Hello"}]
|
| 97 |
+
|
| 98 |
+
# Act
|
| 99 |
+
# Use mocked responses, not real API calls
|
| 100 |
+
|
| 101 |
+
# Assert
|
| 102 |
+
assert expected_result == actual_result
|
| 103 |
+
```
|
| 104 |
+
|
| 105 |
+
## Running Tests and Checks
|
| 106 |
+
|
| 107 |
+
### Running Unit Tests
|
| 108 |
+
|
| 109 |
+
Run all unit tests (uses parallel execution for speed):
|
| 110 |
+
|
| 111 |
+
```bash
|
| 112 |
+
make test-unit
|
| 113 |
+
```
|
| 114 |
+
|
| 115 |
+
Run specific test files:
|
| 116 |
+
```bash
|
| 117 |
+
poetry run pytest tests/test_litellm/test_your_file.py -v
|
| 118 |
+
```
|
| 119 |
+
|
| 120 |
+
### Running Linting and Formatting Checks
|
| 121 |
+
|
| 122 |
+
Run all linting checks (matches CI exactly):
|
| 123 |
+
|
| 124 |
+
```bash
|
| 125 |
+
make lint
|
| 126 |
+
```
|
| 127 |
+
|
| 128 |
+
Individual linting commands:
|
| 129 |
+
```bash
|
| 130 |
+
make format-check # Check Black formatting
|
| 131 |
+
make lint-ruff # Run Ruff linting
|
| 132 |
+
make lint-mypy # Run MyPy type checking
|
| 133 |
+
make check-circular-imports # Check for circular imports
|
| 134 |
+
make check-import-safety # Check import safety
|
| 135 |
+
```
|
| 136 |
+
|
| 137 |
+
Apply formatting (auto-fixes issues):
|
| 138 |
+
```bash
|
| 139 |
+
make format
|
| 140 |
+
```
|
| 141 |
+
|
| 142 |
+
### CI Compatibility
|
| 143 |
+
|
| 144 |
+
To ensure your changes will pass CI, run the exact same checks locally:
|
| 145 |
+
|
| 146 |
+
```bash
|
| 147 |
+
# This runs the same checks as the GitHub workflows
|
| 148 |
+
make lint
|
| 149 |
+
make test-unit
|
| 150 |
+
```
|
| 151 |
+
|
| 152 |
+
For exact CI compatibility (pins OpenAI version like CI):
|
| 153 |
+
```bash
|
| 154 |
+
make install-dev-ci # Installs exact CI dependencies
|
| 155 |
+
```
|
| 156 |
+
|
| 157 |
+
## Available Make Commands
|
| 158 |
+
|
| 159 |
+
Run `make help` to see all available commands:
|
| 160 |
+
|
| 161 |
+
```bash
|
| 162 |
+
make help # Show all available commands
|
| 163 |
+
make install-dev # Install development dependencies
|
| 164 |
+
make install-proxy-dev # Install proxy development dependencies
|
| 165 |
+
make install-test-deps # Install test dependencies (for running tests)
|
| 166 |
+
make format # Apply Black code formatting
|
| 167 |
+
make format-check # Check Black formatting (matches CI)
|
| 168 |
+
make lint # Run all linting checks
|
| 169 |
+
make test-unit # Run unit tests
|
| 170 |
+
make test-integration # Run integration tests
|
| 171 |
+
make test-unit-helm # Run Helm unit tests
|
| 172 |
+
```
|
| 173 |
+
|
| 174 |
+
## Code Quality Standards
|
| 175 |
+
|
| 176 |
+
LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
|
| 177 |
+
|
| 178 |
+
Our automated quality checks include:
|
| 179 |
+
- **Black** for consistent code formatting
|
| 180 |
+
- **Ruff** for linting and code quality
|
| 181 |
+
- **MyPy** for static type checking
|
| 182 |
+
- **Circular import detection**
|
| 183 |
+
- **Import safety validation**
|
| 184 |
+
|
| 185 |
+
All checks must pass before your PR can be merged.
|
| 186 |
+
|
| 187 |
+
## Common Issues and Solutions
|
| 188 |
+
|
| 189 |
+
### 1. Linting Failures
|
| 190 |
+
|
| 191 |
+
If `make lint` fails:
|
| 192 |
+
|
| 193 |
+
1. **Formatting issues**: Run `make format` to auto-fix
|
| 194 |
+
2. **Ruff issues**: Check the output and fix manually
|
| 195 |
+
3. **MyPy issues**: Add proper type hints
|
| 196 |
+
4. **Circular imports**: Refactor import dependencies
|
| 197 |
+
5. **Import safety**: Fix any unprotected imports
|
| 198 |
+
|
| 199 |
+
### 2. Test Failures
|
| 200 |
+
|
| 201 |
+
If `make test-unit` fails:
|
| 202 |
+
|
| 203 |
+
1. Check if you broke existing functionality
|
| 204 |
+
2. Add tests for your new code
|
| 205 |
+
3. Ensure tests use mocks, not real API calls
|
| 206 |
+
4. Check test file naming conventions
|
| 207 |
+
|
| 208 |
+
### 3. Common Development Tips
|
| 209 |
+
|
| 210 |
+
- **Use type hints**: MyPy requires proper type annotations
|
| 211 |
+
- **Write descriptive commit messages**: Help reviewers understand your changes
|
| 212 |
+
- **Keep PRs focused**: One feature/fix per PR
|
| 213 |
+
- **Test edge cases**: Don't just test the happy path
|
| 214 |
+
- **Update documentation**: If you change APIs, update docs
|
| 215 |
+
|
| 216 |
+
## Building and Running Locally
|
| 217 |
+
|
| 218 |
+
### LiteLLM Proxy Server
|
| 219 |
+
|
| 220 |
+
To run the proxy server locally:
|
| 221 |
+
|
| 222 |
+
```bash
|
| 223 |
+
# Install proxy dependencies
|
| 224 |
+
make install-proxy-dev
|
| 225 |
+
|
| 226 |
+
# Start the proxy server
|
| 227 |
+
poetry run litellm --config your_config.yaml
|
| 228 |
+
```
|
| 229 |
+
|
| 230 |
+
### Docker Development
|
| 231 |
+
|
| 232 |
+
If you want to build the Docker image yourself:
|
| 233 |
+
|
| 234 |
+
```bash
|
| 235 |
+
# Build using the non-root Dockerfile
|
| 236 |
+
docker build -f docker/Dockerfile.non_root -t litellm_dev .
|
| 237 |
+
|
| 238 |
+
# Run with your config
|
| 239 |
+
docker run \
|
| 240 |
+
-v $(pwd)/proxy_config.yaml:/app/config.yaml \
|
| 241 |
+
-e LITELLM_MASTER_KEY="sk-1234" \
|
| 242 |
+
-p 4000:4000 \
|
| 243 |
+
litellm_dev \
|
| 244 |
+
--config /app/config.yaml --detailed_debug
|
| 245 |
+
```
|
| 246 |
+
|
| 247 |
+
## Submitting Your PR
|
| 248 |
+
|
| 249 |
+
1. **Push your branch**: `git push origin your-feature-branch`
|
| 250 |
+
2. **Create a PR**: Go to GitHub and create a pull request
|
| 251 |
+
3. **Fill out the PR template**: Provide clear description of changes
|
| 252 |
+
4. **Wait for review**: Maintainers will review and provide feedback
|
| 253 |
+
5. **Address feedback**: Make requested changes and push updates
|
| 254 |
+
6. **Merge**: Once approved, your PR will be merged!
|
| 255 |
+
|
| 256 |
+
## Getting Help
|
| 257 |
+
|
| 258 |
+
If you need help:
|
| 259 |
+
|
| 260 |
+
- 💬 [Join our Discord](https://discord.gg/wuPM9dRgDw)
|
| 261 |
+
- 📧 Email us: [email protected] / [email protected]
|
| 262 |
+
- 🐛 [Create an issue](https://github.com/BerriAI/litellm/issues/new)
|
| 263 |
+
|
| 264 |
+
## What to Contribute
|
| 265 |
+
|
| 266 |
+
Looking for ideas? Check out:
|
| 267 |
+
|
| 268 |
+
- 🐛 [Good first issues](https://github.com/BerriAI/litellm/labels/good%20first%20issue)
|
| 269 |
+
- 🚀 [Feature requests](https://github.com/BerriAI/litellm/labels/enhancement)
|
| 270 |
+
- 📚 Documentation improvements
|
| 271 |
+
- 🧪 Test coverage improvements
|
| 272 |
+
- 🔌 New LLM provider integrations
|
| 273 |
+
|
| 274 |
+
Thank you for contributing to LiteLLM! 🚀
|
Dockerfile
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Base image for building
|
| 2 |
+
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
|
| 3 |
+
|
| 4 |
+
# Runtime image
|
| 5 |
+
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
|
| 6 |
+
# Builder stage
|
| 7 |
+
FROM $LITELLM_BUILD_IMAGE AS builder
|
| 8 |
+
|
| 9 |
+
# Set the working directory to /app
|
| 10 |
+
WORKDIR /app
|
| 11 |
+
|
| 12 |
+
USER root
|
| 13 |
+
|
| 14 |
+
# Install build dependencies
|
| 15 |
+
RUN apk add --no-cache gcc python3-dev openssl openssl-dev
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
RUN pip install --upgrade pip && \
|
| 19 |
+
pip install build
|
| 20 |
+
|
| 21 |
+
# Copy the current directory contents into the container at /app
|
| 22 |
+
COPY . .
|
| 23 |
+
|
| 24 |
+
# Build Admin UI
|
| 25 |
+
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
| 26 |
+
|
| 27 |
+
# Build the package
|
| 28 |
+
RUN rm -rf dist/* && python -m build
|
| 29 |
+
|
| 30 |
+
# There should be only one wheel file now, assume the build only creates one
|
| 31 |
+
RUN ls -1 dist/*.whl | head -1
|
| 32 |
+
|
| 33 |
+
# Install the package
|
| 34 |
+
RUN pip install dist/*.whl
|
| 35 |
+
|
| 36 |
+
# install dependencies as wheels
|
| 37 |
+
RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
|
| 38 |
+
|
| 39 |
+
# ensure pyjwt is used, not jwt
|
| 40 |
+
RUN pip uninstall jwt -y
|
| 41 |
+
RUN pip uninstall PyJWT -y
|
| 42 |
+
RUN pip install PyJWT==2.9.0 --no-cache-dir
|
| 43 |
+
|
| 44 |
+
# Build Admin UI
|
| 45 |
+
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
|
| 46 |
+
|
| 47 |
+
# Runtime stage
|
| 48 |
+
FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
| 49 |
+
|
| 50 |
+
# Ensure runtime stage runs as root
|
| 51 |
+
USER root
|
| 52 |
+
|
| 53 |
+
# Install runtime dependencies
|
| 54 |
+
RUN apk add --no-cache openssl tzdata
|
| 55 |
+
|
| 56 |
+
WORKDIR /app
|
| 57 |
+
# Copy the current directory contents into the container at /app
|
| 58 |
+
COPY . .
|
| 59 |
+
RUN ls -la /app
|
| 60 |
+
|
| 61 |
+
# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
|
| 62 |
+
COPY --from=builder /app/dist/*.whl .
|
| 63 |
+
COPY --from=builder /wheels/ /wheels/
|
| 64 |
+
|
| 65 |
+
# Install the built wheel using pip; again using a wildcard if it's the only file
|
| 66 |
+
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
|
| 67 |
+
|
| 68 |
+
# Generate prisma client
|
| 69 |
+
RUN prisma generate
|
| 70 |
+
RUN chmod +x docker/entrypoint.sh
|
| 71 |
+
RUN chmod +x docker/prod_entrypoint.sh
|
| 72 |
+
|
| 73 |
+
EXPOSE 4000/tcp
|
| 74 |
+
|
| 75 |
+
ENTRYPOINT ["docker/prod_entrypoint.sh"]
|
| 76 |
+
|
| 77 |
+
# Append "--detailed_debug" to the end of CMD to view detailed debug logs
|
| 78 |
+
CMD ["--port", "4000"]
|
LICENSE
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Portions of this software are licensed as follows:
|
| 2 |
+
|
| 3 |
+
* All content that resides under the "enterprise/" directory of this repository, if that directory exists, is licensed under the license defined in "enterprise/LICENSE".
|
| 4 |
+
* Content outside of the above mentioned directories or restrictions above is available under the MIT license as defined below.
|
| 5 |
+
---
|
| 6 |
+
MIT License
|
| 7 |
+
|
| 8 |
+
Copyright (c) 2023 Berri AI
|
| 9 |
+
|
| 10 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 11 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 12 |
+
in the Software without restriction, including without limitation the rights
|
| 13 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 14 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 15 |
+
furnished to do so, subject to the following conditions:
|
| 16 |
+
|
| 17 |
+
The above copyright notice and this permission notice shall be included in all
|
| 18 |
+
copies or substantial portions of the Software.
|
| 19 |
+
|
| 20 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 21 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 22 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 23 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 24 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 25 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 26 |
+
SOFTWARE.
|
Makefile
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# LiteLLM Makefile
|
| 2 |
+
# Simple Makefile for running tests and basic development tasks
|
| 3 |
+
|
| 4 |
+
.PHONY: help test test-unit test-integration test-unit-helm lint format install-dev install-proxy-dev install-test-deps install-helm-unittest check-circular-imports check-import-safety
|
| 5 |
+
|
| 6 |
+
# Default target
|
| 7 |
+
help:
|
| 8 |
+
@echo "Available commands:"
|
| 9 |
+
@echo " make install-dev - Install development dependencies"
|
| 10 |
+
@echo " make install-proxy-dev - Install proxy development dependencies"
|
| 11 |
+
@echo " make install-dev-ci - Install dev dependencies (CI-compatible, pins OpenAI)"
|
| 12 |
+
@echo " make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)"
|
| 13 |
+
@echo " make install-test-deps - Install test dependencies"
|
| 14 |
+
@echo " make install-helm-unittest - Install helm unittest plugin"
|
| 15 |
+
@echo " make format - Apply Black code formatting"
|
| 16 |
+
@echo " make format-check - Check Black code formatting (matches CI)"
|
| 17 |
+
@echo " make lint - Run all linting (Ruff, MyPy, Black check, circular imports, import safety)"
|
| 18 |
+
@echo " make lint-ruff - Run Ruff linting only"
|
| 19 |
+
@echo " make lint-mypy - Run MyPy type checking only"
|
| 20 |
+
@echo " make lint-black - Check Black formatting (matches CI)"
|
| 21 |
+
@echo " make check-circular-imports - Check for circular imports"
|
| 22 |
+
@echo " make check-import-safety - Check import safety"
|
| 23 |
+
@echo " make test - Run all tests"
|
| 24 |
+
@echo " make test-unit - Run unit tests (tests/test_litellm)"
|
| 25 |
+
@echo " make test-integration - Run integration tests"
|
| 26 |
+
@echo " make test-unit-helm - Run helm unit tests"
|
| 27 |
+
|
| 28 |
+
# Installation targets
|
| 29 |
+
install-dev:
|
| 30 |
+
poetry install --with dev
|
| 31 |
+
|
| 32 |
+
install-proxy-dev:
|
| 33 |
+
poetry install --with dev,proxy-dev --extras proxy
|
| 34 |
+
|
| 35 |
+
# CI-compatible installations (matches GitHub workflows exactly)
|
| 36 |
+
install-dev-ci:
|
| 37 |
+
pip install openai==1.81.0
|
| 38 |
+
poetry install --with dev
|
| 39 |
+
pip install openai==1.81.0
|
| 40 |
+
|
| 41 |
+
install-proxy-dev-ci:
|
| 42 |
+
poetry install --with dev,proxy-dev --extras proxy
|
| 43 |
+
pip install openai==1.81.0
|
| 44 |
+
|
| 45 |
+
install-test-deps: install-proxy-dev
|
| 46 |
+
poetry run pip install "pytest-retry==1.6.3"
|
| 47 |
+
poetry run pip install pytest-xdist
|
| 48 |
+
cd enterprise && python -m pip install -e . && cd ..
|
| 49 |
+
|
| 50 |
+
install-helm-unittest:
|
| 51 |
+
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
|
| 52 |
+
|
| 53 |
+
# Formatting
|
| 54 |
+
format: install-dev
|
| 55 |
+
cd litellm && poetry run black . && cd ..
|
| 56 |
+
|
| 57 |
+
format-check: install-dev
|
| 58 |
+
cd litellm && poetry run black --check . && cd ..
|
| 59 |
+
|
| 60 |
+
# Linting targets
|
| 61 |
+
lint-ruff: install-dev
|
| 62 |
+
cd litellm && poetry run ruff check . && cd ..
|
| 63 |
+
|
| 64 |
+
lint-mypy: install-dev
|
| 65 |
+
poetry run pip install types-requests types-setuptools types-redis types-PyYAML
|
| 66 |
+
cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
|
| 67 |
+
|
| 68 |
+
lint-black: format-check
|
| 69 |
+
|
| 70 |
+
check-circular-imports: install-dev
|
| 71 |
+
cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
|
| 72 |
+
|
| 73 |
+
check-import-safety: install-dev
|
| 74 |
+
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
|
| 75 |
+
|
| 76 |
+
# Combined linting (matches test-linting.yml workflow)
|
| 77 |
+
lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
|
| 78 |
+
|
| 79 |
+
# Testing targets
|
| 80 |
+
test:
|
| 81 |
+
poetry run pytest tests/
|
| 82 |
+
|
| 83 |
+
test-unit: install-test-deps
|
| 84 |
+
poetry run pytest tests/test_litellm -x -vv -n 4
|
| 85 |
+
|
| 86 |
+
test-integration:
|
| 87 |
+
poetry run pytest tests/ -k "not test_litellm"
|
| 88 |
+
|
| 89 |
+
test-unit-helm: install-helm-unittest
|
| 90 |
+
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
|
README.md
CHANGED
|
@@ -1,12 +1,441 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<h1 align="center">
|
| 2 |
+
🚅 LiteLLM
|
| 3 |
+
</h1>
|
| 4 |
+
<p align="center">
|
| 5 |
+
<p align="center">
|
| 6 |
+
<a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
|
| 7 |
+
<a href="https://railway.app/template/HLP0Ub?referralCode=jch2ME">
|
| 8 |
+
<img src="https://railway.app/button.svg" alt="Deploy on Railway">
|
| 9 |
+
</a>
|
| 10 |
+
</p>
|
| 11 |
+
<p align="center">Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]
|
| 12 |
+
<br>
|
| 13 |
+
</p>
|
| 14 |
+
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (LLM Gateway)</a> | <a href="https://docs.litellm.ai/docs/hosted" target="_blank"> Hosted Proxy (Preview)</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
|
| 15 |
+
<h4 align="center">
|
| 16 |
+
<a href="https://pypi.org/project/litellm/" target="_blank">
|
| 17 |
+
<img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
|
| 18 |
+
</a>
|
| 19 |
+
<a href="https://www.ycombinator.com/companies/berriai">
|
| 20 |
+
<img src="https://img.shields.io/badge/Y%20Combinator-W23-orange?style=flat-square" alt="Y Combinator W23">
|
| 21 |
+
</a>
|
| 22 |
+
<a href="https://wa.link/huol9n">
|
| 23 |
+
<img src="https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square" alt="Whatsapp">
|
| 24 |
+
</a>
|
| 25 |
+
<a href="https://discord.gg/wuPM9dRgDw">
|
| 26 |
+
<img src="https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square" alt="Discord">
|
| 27 |
+
</a>
|
| 28 |
+
</h4>
|
| 29 |
+
|
| 30 |
+
LiteLLM manages:
|
| 31 |
+
|
| 32 |
+
- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
|
| 33 |
+
- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
|
| 34 |
+
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
|
| 35 |
+
- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
|
| 36 |
+
|
| 37 |
+
[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) <br>
|
| 38 |
+
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
|
| 39 |
+
|
| 40 |
+
🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
|
| 41 |
+
|
| 42 |
+
Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
|
| 43 |
+
|
| 44 |
+
# Usage ([**Docs**](https://docs.litellm.ai/docs/))
|
| 45 |
+
|
| 46 |
+
> [!IMPORTANT]
|
| 47 |
+
> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
|
| 48 |
+
> LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
|
| 49 |
+
|
| 50 |
+
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
|
| 51 |
+
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
|
| 52 |
+
</a>
|
| 53 |
+
|
| 54 |
+
```shell
|
| 55 |
+
pip install litellm
|
| 56 |
+
```
|
| 57 |
+
|
| 58 |
+
```python
|
| 59 |
+
from litellm import completion
|
| 60 |
+
import os
|
| 61 |
+
|
| 62 |
+
## set ENV variables
|
| 63 |
+
os.environ["OPENAI_API_KEY"] = "your-openai-key"
|
| 64 |
+
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
|
| 65 |
+
|
| 66 |
+
messages = [{ "content": "Hello, how are you?","role": "user"}]
|
| 67 |
+
|
| 68 |
+
# openai call
|
| 69 |
+
response = completion(model="openai/gpt-4o", messages=messages)
|
| 70 |
+
|
| 71 |
+
# anthropic call
|
| 72 |
+
response = completion(model="anthropic/claude-3-sonnet-20240229", messages=messages)
|
| 73 |
+
print(response)
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
### Response (OpenAI Format)
|
| 77 |
+
|
| 78 |
+
```json
|
| 79 |
+
{
|
| 80 |
+
"id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885",
|
| 81 |
+
"created": 1734366691,
|
| 82 |
+
"model": "claude-3-sonnet-20240229",
|
| 83 |
+
"object": "chat.completion",
|
| 84 |
+
"system_fingerprint": null,
|
| 85 |
+
"choices": [
|
| 86 |
+
{
|
| 87 |
+
"finish_reason": "stop",
|
| 88 |
+
"index": 0,
|
| 89 |
+
"message": {
|
| 90 |
+
"content": "Hello! As an AI language model, I don't have feelings, but I'm operating properly and ready to assist you with any questions or tasks you may have. How can I help you today?",
|
| 91 |
+
"role": "assistant",
|
| 92 |
+
"tool_calls": null,
|
| 93 |
+
"function_call": null
|
| 94 |
+
}
|
| 95 |
+
}
|
| 96 |
+
],
|
| 97 |
+
"usage": {
|
| 98 |
+
"completion_tokens": 43,
|
| 99 |
+
"prompt_tokens": 13,
|
| 100 |
+
"total_tokens": 56,
|
| 101 |
+
"completion_tokens_details": null,
|
| 102 |
+
"prompt_tokens_details": {
|
| 103 |
+
"audio_tokens": null,
|
| 104 |
+
"cached_tokens": 0
|
| 105 |
+
},
|
| 106 |
+
"cache_creation_input_tokens": 0,
|
| 107 |
+
"cache_read_input_tokens": 0
|
| 108 |
+
}
|
| 109 |
+
}
|
| 110 |
+
```
|
| 111 |
+
|
| 112 |
+
Call any model supported by a provider, with `model=<provider_name>/<model_name>`. There might be provider-specific details here, so refer to [provider docs for more information](https://docs.litellm.ai/docs/providers)
|
| 113 |
+
|
| 114 |
+
## Async ([Docs](https://docs.litellm.ai/docs/completion/stream#async-completion))
|
| 115 |
+
|
| 116 |
+
```python
|
| 117 |
+
from litellm import acompletion
|
| 118 |
+
import asyncio
|
| 119 |
+
|
| 120 |
+
async def test_get_response():
|
| 121 |
+
user_message = "Hello, how are you?"
|
| 122 |
+
messages = [{"content": user_message, "role": "user"}]
|
| 123 |
+
response = await acompletion(model="openai/gpt-4o", messages=messages)
|
| 124 |
+
return response
|
| 125 |
+
|
| 126 |
+
response = asyncio.run(test_get_response())
|
| 127 |
+
print(response)
|
| 128 |
+
```
|
| 129 |
+
|
| 130 |
+
## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
|
| 131 |
+
|
| 132 |
+
liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
|
| 133 |
+
Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
|
| 134 |
+
|
| 135 |
+
```python
|
| 136 |
+
from litellm import completion
|
| 137 |
+
response = completion(model="openai/gpt-4o", messages=messages, stream=True)
|
| 138 |
+
for part in response:
|
| 139 |
+
print(part.choices[0].delta.content or "")
|
| 140 |
+
|
| 141 |
+
# claude 2
|
| 142 |
+
response = completion('anthropic/claude-3-sonnet-20240229', messages, stream=True)
|
| 143 |
+
for part in response:
|
| 144 |
+
print(part)
|
| 145 |
+
```
|
| 146 |
+
|
| 147 |
+
### Response chunk (OpenAI Format)
|
| 148 |
+
|
| 149 |
+
```json
|
| 150 |
+
{
|
| 151 |
+
"id": "chatcmpl-2be06597-eb60-4c70-9ec5-8cd2ab1b4697",
|
| 152 |
+
"created": 1734366925,
|
| 153 |
+
"model": "claude-3-sonnet-20240229",
|
| 154 |
+
"object": "chat.completion.chunk",
|
| 155 |
+
"system_fingerprint": null,
|
| 156 |
+
"choices": [
|
| 157 |
+
{
|
| 158 |
+
"finish_reason": null,
|
| 159 |
+
"index": 0,
|
| 160 |
+
"delta": {
|
| 161 |
+
"content": "Hello",
|
| 162 |
+
"role": "assistant",
|
| 163 |
+
"function_call": null,
|
| 164 |
+
"tool_calls": null,
|
| 165 |
+
"audio": null
|
| 166 |
+
},
|
| 167 |
+
"logprobs": null
|
| 168 |
+
}
|
| 169 |
+
]
|
| 170 |
+
}
|
| 171 |
+
```
|
| 172 |
+
|
| 173 |
+
## Logging Observability ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
|
| 174 |
+
|
| 175 |
+
LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack
|
| 176 |
+
|
| 177 |
+
```python
|
| 178 |
+
from litellm import completion
|
| 179 |
+
|
| 180 |
+
## set env variables for logging tools (when using MLflow, no API key set up is required)
|
| 181 |
+
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
|
| 182 |
+
os.environ["HELICONE_API_KEY"] = "your-helicone-auth-key"
|
| 183 |
+
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
|
| 184 |
+
os.environ["LANGFUSE_SECRET_KEY"] = ""
|
| 185 |
+
os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
|
| 186 |
+
|
| 187 |
+
os.environ["OPENAI_API_KEY"] = "your-openai-key"
|
| 188 |
+
|
| 189 |
+
# set callbacks
|
| 190 |
+
litellm.success_callback = ["lunary", "mlflow", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
|
| 191 |
+
|
| 192 |
+
#openai call
|
| 193 |
+
response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
|
| 194 |
+
```
|
| 195 |
+
|
| 196 |
+
# LiteLLM Proxy Server (LLM Gateway) - ([Docs](https://docs.litellm.ai/docs/simple_proxy))
|
| 197 |
+
|
| 198 |
+
Track spend + Load Balance across multiple projects
|
| 199 |
+
|
| 200 |
+
[Hosted Proxy (Preview)](https://docs.litellm.ai/docs/hosted)
|
| 201 |
+
|
| 202 |
+
The proxy provides:
|
| 203 |
+
|
| 204 |
+
1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth)
|
| 205 |
+
2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class)
|
| 206 |
+
3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend)
|
| 207 |
+
4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits)
|
| 208 |
+
|
| 209 |
+
## 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/)
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
## Quick Start Proxy - CLI
|
| 213 |
+
|
| 214 |
+
```shell
|
| 215 |
+
pip install 'litellm[proxy]'
|
| 216 |
+
```
|
| 217 |
+
|
| 218 |
+
### Step 1: Start litellm proxy
|
| 219 |
+
|
| 220 |
+
```shell
|
| 221 |
+
$ litellm --model huggingface/bigcode/starcoder
|
| 222 |
+
|
| 223 |
+
#INFO: Proxy running on http://0.0.0.0:4000
|
| 224 |
+
```
|
| 225 |
+
|
| 226 |
+
### Step 2: Make ChatCompletions Request to Proxy
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
> [!IMPORTANT]
|
| 230 |
+
> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
|
| 231 |
+
|
| 232 |
+
```python
|
| 233 |
+
import openai # openai v1.0.0+
|
| 234 |
+
client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url
|
| 235 |
+
# request sent to model set on litellm proxy, `litellm --model`
|
| 236 |
+
response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
|
| 237 |
+
{
|
| 238 |
+
"role": "user",
|
| 239 |
+
"content": "this is a test request, write a short poem"
|
| 240 |
+
}
|
| 241 |
+
])
|
| 242 |
+
|
| 243 |
+
print(response)
|
| 244 |
+
```
|
| 245 |
+
|
| 246 |
+
## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys))
|
| 247 |
+
|
| 248 |
+
Connect the proxy with a Postgres DB to create proxy keys
|
| 249 |
+
|
| 250 |
+
```bash
|
| 251 |
+
# Get the code
|
| 252 |
+
git clone https://github.com/BerriAI/litellm
|
| 253 |
+
|
| 254 |
+
# Go to folder
|
| 255 |
+
cd litellm
|
| 256 |
+
|
| 257 |
+
# Add the master key - you can change this after setup
|
| 258 |
+
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
|
| 259 |
+
|
| 260 |
+
# Add the litellm salt key - you cannot change this after adding a model
|
| 261 |
+
# It is used to encrypt / decrypt your LLM API Key credentials
|
| 262 |
+
# We recommend - https://1password.com/password-generator/
|
| 263 |
+
# password generator to get a random hash for litellm salt key
|
| 264 |
+
echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
|
| 265 |
+
|
| 266 |
+
source .env
|
| 267 |
+
|
| 268 |
+
# Start
|
| 269 |
+
docker-compose up
|
| 270 |
+
```
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
UI on `/ui` on your proxy server
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
Set budgets and rate limits across multiple projects
|
| 277 |
+
`POST /key/generate`
|
| 278 |
+
|
| 279 |
+
### Request
|
| 280 |
+
|
| 281 |
+
```shell
|
| 282 |
+
curl 'http://0.0.0.0:4000/key/generate' \
|
| 283 |
+
--header 'Authorization: Bearer sk-1234' \
|
| 284 |
+
--header 'Content-Type: application/json' \
|
| 285 |
+
--data-raw '{"models": ["gpt-3.5-turbo", "gpt-4", "claude-2"], "duration": "20m","metadata": {"user": "[email protected]", "team": "core-infra"}}'
|
| 286 |
+
```
|
| 287 |
+
|
| 288 |
+
### Expected Response
|
| 289 |
+
|
| 290 |
+
```shell
|
| 291 |
+
{
|
| 292 |
+
"key": "sk-kdEXbIqZRwEeEiHwdg7sFA", # Bearer token
|
| 293 |
+
"expires": "2023-11-19T01:38:25.838000+00:00" # datetime object
|
| 294 |
+
}
|
| 295 |
+
```
|
| 296 |
+
|
| 297 |
+
## Supported Providers ([Docs](https://docs.litellm.ai/docs/providers))
|
| 298 |
+
|
| 299 |
+
| Provider | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) | [Async Image Generation](https://docs.litellm.ai/docs/image_generation) |
|
| 300 |
+
|-------------------------------------------------------------------------------------|---------------------------------------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------|-------------------------------------------------------------------------------|-------------------------------------------------------------------------|
|
| 301 |
+
| [openai](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
| 302 |
+
| [Meta - Llama API](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ | ✅ | | |
|
| 303 |
+
| [azure](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
| 304 |
+
| [AI/ML API](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
| 305 |
+
| [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
| 306 |
+
| [aws - bedrock](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
| 307 |
+
| [google - vertex_ai](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
|
| 308 |
+
| [google - palm](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | ✅ | | |
|
| 309 |
+
| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | | |
|
| 310 |
+
| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
| 311 |
+
| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ | | |
|
| 312 |
+
| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
| 313 |
+
| [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ | | |
|
| 314 |
+
| [empower](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | ✅ |
|
| 315 |
+
| [huggingface](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
| 316 |
+
| [replicate](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | ✅ | | |
|
| 317 |
+
| [together_ai](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | ✅ | | |
|
| 318 |
+
| [openrouter](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | ✅ | | |
|
| 319 |
+
| [ai21](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | ✅ | | |
|
| 320 |
+
| [baseten](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | ✅ | | |
|
| 321 |
+
| [vllm](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | ✅ | | |
|
| 322 |
+
| [nlp_cloud](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | ✅ | | |
|
| 323 |
+
| [aleph alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | ✅ | | |
|
| 324 |
+
| [petals](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | ✅ | | |
|
| 325 |
+
| [ollama](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
| 326 |
+
| [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ | | |
|
| 327 |
+
| [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ | | |
|
| 328 |
+
| [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ | | |
|
| 329 |
+
| [Deepseek](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | ✅ | | |
|
| 330 |
+
| [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ | | |
|
| 331 |
+
| [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
| 332 |
+
| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ | |
|
| 333 |
+
| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | |
|
| 334 |
+
| [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | ✅ | | |
|
| 335 |
+
| [Galadriel](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | ✅ | | |
|
| 336 |
+
| [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | ✅ | | |
|
| 337 |
+
| [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | ✅ | | |
|
| 338 |
+
| [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | ✅ | |
|
| 339 |
+
|
| 340 |
+
[**Read the Docs**](https://docs.litellm.ai/docs/)
|
| 341 |
+
|
| 342 |
+
## Contributing
|
| 343 |
+
|
| 344 |
+
Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and LLM integrations are both accepted and highly encouraged!
|
| 345 |
+
|
| 346 |
+
**Quick start:** `git clone` → `make install-dev` → `make format` → `make lint` → `make test-unit`
|
| 347 |
+
|
| 348 |
+
See our comprehensive [Contributing Guide (CONTRIBUTING.md)](CONTRIBUTING.md) for detailed instructions.
|
| 349 |
+
|
| 350 |
+
# Enterprise
|
| 351 |
+
For companies that need better security, user management and professional support
|
| 352 |
+
|
| 353 |
+
[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
|
| 354 |
+
|
| 355 |
+
This covers:
|
| 356 |
+
- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
|
| 357 |
+
- ✅ **Feature Prioritization**
|
| 358 |
+
- ✅ **Custom Integrations**
|
| 359 |
+
- ✅ **Professional Support - Dedicated discord + slack**
|
| 360 |
+
- ✅ **Custom SLAs**
|
| 361 |
+
- ✅ **Secure access with Single Sign-On**
|
| 362 |
+
|
| 363 |
+
# Contributing
|
| 364 |
+
|
| 365 |
+
We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features, or improving documentation, we appreciate your help.
|
| 366 |
+
|
| 367 |
+
## Quick Start for Contributors
|
| 368 |
+
|
| 369 |
+
```bash
|
| 370 |
+
git clone https://github.com/BerriAI/litellm.git
|
| 371 |
+
cd litellm
|
| 372 |
+
make install-dev # Install development dependencies
|
| 373 |
+
make format # Format your code
|
| 374 |
+
make lint # Run all linting checks
|
| 375 |
+
make test-unit # Run unit tests
|
| 376 |
+
```
|
| 377 |
+
|
| 378 |
+
For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
|
| 379 |
+
|
| 380 |
+
## Code Quality / Linting
|
| 381 |
+
|
| 382 |
+
LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
|
| 383 |
+
|
| 384 |
+
Our automated checks include:
|
| 385 |
+
- **Black** for code formatting
|
| 386 |
+
- **Ruff** for linting and code quality
|
| 387 |
+
- **MyPy** for type checking
|
| 388 |
+
- **Circular import detection**
|
| 389 |
+
- **Import safety checks**
|
| 390 |
+
|
| 391 |
+
Run all checks locally:
|
| 392 |
+
```bash
|
| 393 |
+
make lint # Run all linting (matches CI)
|
| 394 |
+
make format-check # Check formatting only
|
| 395 |
+
```
|
| 396 |
+
|
| 397 |
+
All these checks must pass before your PR can be merged.
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
# Support / talk with founders
|
| 401 |
+
|
| 402 |
+
- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
|
| 403 |
+
- [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
|
| 404 |
+
- Our numbers 📞 +1 (770) 8783-106 / +1 (412) 618-6238
|
| 405 |
+
- Our emails ✉️ [email protected] / [email protected]
|
| 406 |
+
|
| 407 |
+
# Why did we build this
|
| 408 |
+
|
| 409 |
+
- **Need for simplicity**: Our code started to get extremely complicated managing & translating calls between Azure, OpenAI and Cohere.
|
| 410 |
+
|
| 411 |
+
# Contributors
|
| 412 |
+
|
| 413 |
+
<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
|
| 414 |
+
<!-- prettier-ignore-start -->
|
| 415 |
+
<!-- markdownlint-disable -->
|
| 416 |
+
|
| 417 |
+
<!-- markdownlint-restore -->
|
| 418 |
+
<!-- prettier-ignore-end -->
|
| 419 |
+
|
| 420 |
+
<!-- ALL-CONTRIBUTORS-LIST:END -->
|
| 421 |
+
|
| 422 |
+
<a href="https://github.com/BerriAI/litellm/graphs/contributors">
|
| 423 |
+
<img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
|
| 424 |
+
</a>
|
| 425 |
+
|
| 426 |
+
|
| 427 |
+
## Run in Developer mode
|
| 428 |
+
### Services
|
| 429 |
+
1. Setup .env file in root
|
| 430 |
+
2. Run dependant services `docker-compose up db prometheus`
|
| 431 |
+
|
| 432 |
+
### Backend
|
| 433 |
+
1. (In root) create virtual environment `python -m venv .venv`
|
| 434 |
+
2. Activate virtual environment `source .venv/bin/activate`
|
| 435 |
+
3. Install dependencies `pip install -e ".[all]"`
|
| 436 |
+
4. Start proxy backend `uvicorn litellm.proxy.proxy_server:app --host localhost --port 4000 --reload`
|
| 437 |
+
|
| 438 |
+
### Frontend
|
| 439 |
+
1. Navigate to `ui/litellm-dashboard`
|
| 440 |
+
2. Install dependencies `npm install`
|
| 441 |
+
3. Run `npm run dev` to start the dashboard
|
ci_cd/baseline_db.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import subprocess
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
from datetime import datetime
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def create_baseline():
|
| 7 |
+
"""Create baseline migration in deploy/migrations"""
|
| 8 |
+
try:
|
| 9 |
+
# Get paths
|
| 10 |
+
root_dir = Path(__file__).parent.parent
|
| 11 |
+
deploy_dir = root_dir / "deploy"
|
| 12 |
+
migrations_dir = deploy_dir / "migrations"
|
| 13 |
+
schema_path = root_dir / "schema.prisma"
|
| 14 |
+
|
| 15 |
+
# Create migrations directory
|
| 16 |
+
migrations_dir.mkdir(parents=True, exist_ok=True)
|
| 17 |
+
|
| 18 |
+
# Create migration_lock.toml if it doesn't exist
|
| 19 |
+
lock_file = migrations_dir / "migration_lock.toml"
|
| 20 |
+
if not lock_file.exists():
|
| 21 |
+
lock_file.write_text('provider = "postgresql"\n')
|
| 22 |
+
|
| 23 |
+
# Create timestamp-based migration directory
|
| 24 |
+
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
| 25 |
+
migration_dir = migrations_dir / f"{timestamp}_baseline"
|
| 26 |
+
migration_dir.mkdir(parents=True, exist_ok=True)
|
| 27 |
+
|
| 28 |
+
# Generate migration SQL
|
| 29 |
+
result = subprocess.run(
|
| 30 |
+
[
|
| 31 |
+
"prisma",
|
| 32 |
+
"migrate",
|
| 33 |
+
"diff",
|
| 34 |
+
"--from-empty",
|
| 35 |
+
"--to-schema-datamodel",
|
| 36 |
+
str(schema_path),
|
| 37 |
+
"--script",
|
| 38 |
+
],
|
| 39 |
+
capture_output=True,
|
| 40 |
+
text=True,
|
| 41 |
+
check=True,
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
# Write the SQL to migration.sql
|
| 45 |
+
migration_file = migration_dir / "migration.sql"
|
| 46 |
+
migration_file.write_text(result.stdout)
|
| 47 |
+
|
| 48 |
+
print(f"Created baseline migration in {migration_dir}")
|
| 49 |
+
return True
|
| 50 |
+
|
| 51 |
+
except subprocess.CalledProcessError as e:
|
| 52 |
+
print(f"Error running prisma command: {e.stderr}")
|
| 53 |
+
return False
|
| 54 |
+
except Exception as e:
|
| 55 |
+
print(f"Error creating baseline migration: {str(e)}")
|
| 56 |
+
return False
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
if __name__ == "__main__":
|
| 60 |
+
create_baseline()
|
ci_cd/check_file_length.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import sys
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
def check_file_length(max_lines, filenames):
|
| 5 |
+
bad_files = []
|
| 6 |
+
for filename in filenames:
|
| 7 |
+
with open(filename, "r") as file:
|
| 8 |
+
lines = file.readlines()
|
| 9 |
+
if len(lines) > max_lines:
|
| 10 |
+
bad_files.append((filename, len(lines)))
|
| 11 |
+
return bad_files
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
if __name__ == "__main__":
|
| 15 |
+
max_lines = int(sys.argv[1])
|
| 16 |
+
filenames = sys.argv[2:]
|
| 17 |
+
|
| 18 |
+
bad_files = check_file_length(max_lines, filenames)
|
| 19 |
+
if bad_files:
|
| 20 |
+
bad_files.sort(
|
| 21 |
+
key=lambda x: x[1], reverse=True
|
| 22 |
+
) # Sort files by length in descending order
|
| 23 |
+
for filename, length in bad_files:
|
| 24 |
+
print(f"{filename}: {length} lines")
|
| 25 |
+
|
| 26 |
+
sys.exit(1)
|
| 27 |
+
else:
|
| 28 |
+
sys.exit(0)
|
ci_cd/check_files_match.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import sys
|
| 2 |
+
import filecmp
|
| 3 |
+
import shutil
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def main(argv=None):
|
| 7 |
+
print(
|
| 8 |
+
"Comparing model_prices_and_context_window and litellm/model_prices_and_context_window_backup.json files... checking if they match."
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
file1 = "model_prices_and_context_window.json"
|
| 12 |
+
file2 = "litellm/model_prices_and_context_window_backup.json"
|
| 13 |
+
|
| 14 |
+
cmp_result = filecmp.cmp(file1, file2, shallow=False)
|
| 15 |
+
|
| 16 |
+
if cmp_result:
|
| 17 |
+
print(f"Passed! Files {file1} and {file2} match.")
|
| 18 |
+
return 0
|
| 19 |
+
else:
|
| 20 |
+
print(
|
| 21 |
+
f"Failed! Files {file1} and {file2} do not match. Copying content from {file1} to {file2}."
|
| 22 |
+
)
|
| 23 |
+
copy_content(file1, file2)
|
| 24 |
+
return 1
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def copy_content(source, destination):
|
| 28 |
+
shutil.copy2(source, destination)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
if __name__ == "__main__":
|
| 32 |
+
sys.exit(main())
|
ci_cd/publish-proxy-extras.sh
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
|
| 3 |
+
# Exit on error
|
| 4 |
+
set -e
|
| 5 |
+
|
| 6 |
+
echo "🚀 Building and publishing litellm-proxy-extras"
|
| 7 |
+
|
| 8 |
+
# Navigate to litellm-proxy-extras directory
|
| 9 |
+
cd "$(dirname "$0")/../litellm-proxy-extras"
|
| 10 |
+
|
| 11 |
+
# Build the package
|
| 12 |
+
echo "📦 Building package..."
|
| 13 |
+
poetry build
|
| 14 |
+
|
| 15 |
+
# Publish to PyPI
|
| 16 |
+
echo "🌎 Publishing to PyPI..."
|
| 17 |
+
poetry publish
|
| 18 |
+
|
| 19 |
+
echo "✅ Done! Package published successfully"
|
ci_cd/run_migration.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import subprocess
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from datetime import datetime
|
| 5 |
+
import testing.postgresql
|
| 6 |
+
import shutil
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def create_migration(migration_name: str = None):
|
| 10 |
+
"""
|
| 11 |
+
Create a new migration SQL file in the migrations directory by comparing
|
| 12 |
+
current database state with schema
|
| 13 |
+
|
| 14 |
+
Args:
|
| 15 |
+
migration_name (str): Name for the migration
|
| 16 |
+
"""
|
| 17 |
+
try:
|
| 18 |
+
# Get paths
|
| 19 |
+
root_dir = Path(__file__).parent.parent
|
| 20 |
+
migrations_dir = root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
|
| 21 |
+
schema_path = root_dir / "schema.prisma"
|
| 22 |
+
|
| 23 |
+
# Create temporary PostgreSQL database
|
| 24 |
+
with testing.postgresql.Postgresql() as postgresql:
|
| 25 |
+
db_url = postgresql.url()
|
| 26 |
+
|
| 27 |
+
# Create temporary migrations directory next to schema.prisma
|
| 28 |
+
temp_migrations_dir = schema_path.parent / "migrations"
|
| 29 |
+
|
| 30 |
+
try:
|
| 31 |
+
# Copy existing migrations to temp directory
|
| 32 |
+
if temp_migrations_dir.exists():
|
| 33 |
+
shutil.rmtree(temp_migrations_dir)
|
| 34 |
+
shutil.copytree(migrations_dir, temp_migrations_dir)
|
| 35 |
+
|
| 36 |
+
# Apply existing migrations to temp database
|
| 37 |
+
os.environ["DATABASE_URL"] = db_url
|
| 38 |
+
subprocess.run(
|
| 39 |
+
["prisma", "migrate", "deploy", "--schema", str(schema_path)],
|
| 40 |
+
check=True,
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
# Generate diff between current database and schema
|
| 44 |
+
result = subprocess.run(
|
| 45 |
+
[
|
| 46 |
+
"prisma",
|
| 47 |
+
"migrate",
|
| 48 |
+
"diff",
|
| 49 |
+
"--from-url",
|
| 50 |
+
db_url,
|
| 51 |
+
"--to-schema-datamodel",
|
| 52 |
+
str(schema_path),
|
| 53 |
+
"--script",
|
| 54 |
+
],
|
| 55 |
+
capture_output=True,
|
| 56 |
+
text=True,
|
| 57 |
+
check=True,
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
if result.stdout.strip():
|
| 61 |
+
# Generate timestamp and create migration directory
|
| 62 |
+
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
| 63 |
+
migration_name = migration_name or "unnamed_migration"
|
| 64 |
+
migration_dir = migrations_dir / f"{timestamp}_{migration_name}"
|
| 65 |
+
migration_dir.mkdir(parents=True, exist_ok=True)
|
| 66 |
+
|
| 67 |
+
# Write the SQL to migration.sql
|
| 68 |
+
migration_file = migration_dir / "migration.sql"
|
| 69 |
+
migration_file.write_text(result.stdout)
|
| 70 |
+
|
| 71 |
+
print(f"Created migration in {migration_dir}")
|
| 72 |
+
return True
|
| 73 |
+
else:
|
| 74 |
+
print("No schema changes detected. Migration not needed.")
|
| 75 |
+
return False
|
| 76 |
+
|
| 77 |
+
finally:
|
| 78 |
+
# Clean up: remove temporary migrations directory
|
| 79 |
+
if temp_migrations_dir.exists():
|
| 80 |
+
shutil.rmtree(temp_migrations_dir)
|
| 81 |
+
|
| 82 |
+
except subprocess.CalledProcessError as e:
|
| 83 |
+
print(f"Error generating migration: {e.stderr}")
|
| 84 |
+
return False
|
| 85 |
+
except Exception as e:
|
| 86 |
+
print(f"Error creating migration: {str(e)}")
|
| 87 |
+
return False
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
if __name__ == "__main__":
|
| 91 |
+
# If running directly, can optionally pass migration name as argument
|
| 92 |
+
import sys
|
| 93 |
+
|
| 94 |
+
migration_name = sys.argv[1] if len(sys.argv) > 1 else None
|
| 95 |
+
create_migration(migration_name)
|
codecov.yaml
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
component_management:
|
| 2 |
+
individual_components:
|
| 3 |
+
- component_id: "Router"
|
| 4 |
+
paths:
|
| 5 |
+
- "router"
|
| 6 |
+
- component_id: "LLMs"
|
| 7 |
+
paths:
|
| 8 |
+
- "*/llms/*"
|
| 9 |
+
- component_id: "Caching"
|
| 10 |
+
paths:
|
| 11 |
+
- "*/caching/*"
|
| 12 |
+
- ".*redis.*"
|
| 13 |
+
- component_id: "litellm_logging"
|
| 14 |
+
paths:
|
| 15 |
+
- "*/integrations/*"
|
| 16 |
+
- ".*litellm_logging.*"
|
| 17 |
+
- component_id: "Proxy_Authentication"
|
| 18 |
+
paths:
|
| 19 |
+
- "*/proxy/auth/**"
|
| 20 |
+
comment:
|
| 21 |
+
layout: "header, diff, flags, components" # show component info in the PR comment
|
| 22 |
+
|
| 23 |
+
coverage:
|
| 24 |
+
status:
|
| 25 |
+
project:
|
| 26 |
+
default:
|
| 27 |
+
target: auto
|
| 28 |
+
threshold: 1% # at maximum allow project coverage to drop by 1%
|
| 29 |
+
patch:
|
| 30 |
+
default:
|
| 31 |
+
target: auto
|
| 32 |
+
threshold: 0% # patch coverage should be 100%
|
cookbook/Benchmarking_LLMs_by_use_case.ipynb
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|