Spaces:
Sleeping
Sleeping
More cleanup
Browse files- .buildkite/run-benchmarks.sh +0 -63
- .buildkite/test-pipeline.yaml +0 -51
- .buildkite/test-template.j2 +0 -54
- .dockerignore +0 -1
- .gitattributes +0 -35
- .github/workflows/publish.yml +0 -102
- .github/workflows/ruff.yml +0 -31
- .github/workflows/scripts/build.sh +0 -20
- .github/workflows/scripts/create_release.js +0 -20
- .github/workflows/scripts/cuda-install.sh +0 -23
- .github/workflows/scripts/env.sh +0 -56
- .github/workflows/scripts/pytorch-install.sh +0 -15
- .github/workflows/yapf.yml +0 -31
- .readthedocs.yaml +0 -21
- aiproxy/__main__.py +3 -3
- aiproxy/aiproxy.db +0 -0
- requirements.txt +6 -0
- server.csr +0 -18
- server.key +0 -30
.buildkite/run-benchmarks.sh
DELETED
@@ -1,63 +0,0 @@
|
|
1 |
-
# This script is run by buildkite to run the benchmarks and upload the results to buildkite
|
2 |
-
|
3 |
-
set -ex
|
4 |
-
set -o pipefail
|
5 |
-
|
6 |
-
# cd into parent directory of this file
|
7 |
-
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
8 |
-
|
9 |
-
(wget && curl) || (apt-get update && apt-get install -y wget curl)
|
10 |
-
|
11 |
-
# run benchmarks and upload the result to buildkite
|
12 |
-
python3 benchmarks/benchmark_latency.py 2>&1 | tee benchmark_latency.txt
|
13 |
-
bench_latency_exit_code=$?
|
14 |
-
|
15 |
-
python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 2>&1 | tee benchmark_throughput.txt
|
16 |
-
bench_throughput_exit_code=$?
|
17 |
-
|
18 |
-
python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-chat-hf &
|
19 |
-
server_pid=$!
|
20 |
-
wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
|
21 |
-
|
22 |
-
# wait for server to start, timeout after 600 seconds
|
23 |
-
timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
|
24 |
-
python3 benchmarks/benchmark_serving.py \
|
25 |
-
--dataset ./ShareGPT_V3_unfiltered_cleaned_split.json \
|
26 |
-
--model meta-llama/Llama-2-7b-chat-hf \
|
27 |
-
--num-prompts 20 \
|
28 |
-
--endpoint /v1/completions \
|
29 |
-
--tokenizer meta-llama/Llama-2-7b-chat-hf 2>&1 | tee benchmark_serving.txt
|
30 |
-
bench_serving_exit_code=$?
|
31 |
-
kill $server_pid
|
32 |
-
|
33 |
-
# write the results into a markdown file
|
34 |
-
echo "### Latency Benchmarks" >> benchmark_results.md
|
35 |
-
sed -n '1p' benchmark_latency.txt >> benchmark_results.md # first line
|
36 |
-
echo "" >> benchmark_results.md
|
37 |
-
sed -n '$p' benchmark_latency.txt >> benchmark_results.md # last line
|
38 |
-
|
39 |
-
echo "### Throughput Benchmarks" >> benchmark_results.md
|
40 |
-
sed -n '1p' benchmark_throughput.txt >> benchmark_results.md # first line
|
41 |
-
echo "" >> benchmark_results.md
|
42 |
-
sed -n '$p' benchmark_throughput.txt >> benchmark_results.md # last line
|
43 |
-
|
44 |
-
echo "### Serving Benchmarks" >> benchmark_results.md
|
45 |
-
sed -n '1p' benchmark_serving.txt >> benchmark_results.md # first line
|
46 |
-
echo "" >> benchmark_results.md
|
47 |
-
tail -n 5 benchmark_serving.txt >> benchmark_results.md # last 5 lines
|
48 |
-
|
49 |
-
# upload the results to buildkite
|
50 |
-
/workspace/buildkite-agent annotate --style "info" --context "benchmark-results" < benchmark_results.md
|
51 |
-
|
52 |
-
# exit with the exit code of the benchmarks
|
53 |
-
if [ $bench_latency_exit_code -ne 0 ]; then
|
54 |
-
exit $bench_latency_exit_code
|
55 |
-
fi
|
56 |
-
|
57 |
-
if [ $bench_throughput_exit_code -ne 0 ]; then
|
58 |
-
exit $bench_throughput_exit_code
|
59 |
-
fi
|
60 |
-
|
61 |
-
if [ $bench_serving_exit_code -ne 0 ]; then
|
62 |
-
exit $bench_serving_exit_code
|
63 |
-
fi
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.buildkite/test-pipeline.yaml
DELETED
@@ -1,51 +0,0 @@
|
|
1 |
-
# In this file, you can add more tests to run either by adding a new step or
|
2 |
-
# adding a new command to an existing step. See different options here for examples.
|
3 |
-
# This script will be feed into Jinja template in `test-template.j2` to generate
|
4 |
-
# the final pipeline yaml file.
|
5 |
-
|
6 |
-
steps:
|
7 |
-
- label: Regression Test
|
8 |
-
command: pytest -v -s test_regression.py
|
9 |
-
working_dir: "/vllm-workspace/tests" # optional
|
10 |
-
|
11 |
-
- label: AsyncEngine Test
|
12 |
-
command: pytest -v -s async_engine
|
13 |
-
|
14 |
-
- label: Distributed Test
|
15 |
-
command: pytest -v -s test_comm_ops.py
|
16 |
-
working_dir: "/vllm-workspace/tests/distributed"
|
17 |
-
num_gpus: 2 # only support 1 or 2 for now.
|
18 |
-
|
19 |
-
- label: Engine Test
|
20 |
-
command: pytest -v -s engine
|
21 |
-
|
22 |
-
- label: Entrypoints Test
|
23 |
-
command: pytest -v -s entrypoints
|
24 |
-
|
25 |
-
- label: Kernels Test
|
26 |
-
command: pytest -v -s kernels
|
27 |
-
soft_fail: true
|
28 |
-
|
29 |
-
- label: Models Test
|
30 |
-
commands:
|
31 |
-
- pytest -v -s models --forked
|
32 |
-
soft_fail: true
|
33 |
-
|
34 |
-
- label: Prefix Caching Test
|
35 |
-
commands:
|
36 |
-
- pytest -v -s prefix_caching
|
37 |
-
|
38 |
-
- label: Samplers Test
|
39 |
-
command: pytest -v -s samplers --forked
|
40 |
-
|
41 |
-
- label: Worker Test
|
42 |
-
command: pytest -v -s worker
|
43 |
-
|
44 |
-
- label: LoRA Test
|
45 |
-
command: pytest -v -s lora
|
46 |
-
|
47 |
-
- label: Benchmarks
|
48 |
-
working_dir: "/vllm-workspace/.buildkite"
|
49 |
-
commands:
|
50 |
-
- pip install aiohttp
|
51 |
-
- bash run-benchmarks.sh
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.buildkite/test-template.j2
DELETED
@@ -1,54 +0,0 @@
|
|
1 |
-
{% set docker_image = "us-central1-docker.pkg.dev/vllm-405802/vllm-ci-test-repo/vllm-test:$BUILDKITE_COMMIT" %}
|
2 |
-
{% set default_num_gpu = 1 %}
|
3 |
-
{% set default_working_dir = "/vllm-workspace/tests" %}
|
4 |
-
|
5 |
-
steps:
|
6 |
-
- label: ":docker: build image"
|
7 |
-
commands:
|
8 |
-
- "docker build --build-arg max_jobs=16 --tag {{ docker_image }} --target test --progress plain ."
|
9 |
-
- "docker push {{ docker_image }}"
|
10 |
-
env:
|
11 |
-
DOCKER_BUILDKIT: "1"
|
12 |
-
retry:
|
13 |
-
automatic:
|
14 |
-
- exit_status: -1 # Agent was lost
|
15 |
-
limit: 5
|
16 |
-
- wait
|
17 |
-
|
18 |
-
{% for step in steps %}
|
19 |
-
- label: "{{ step.label }}"
|
20 |
-
agents:
|
21 |
-
queue: kubernetes
|
22 |
-
soft_fail: {{ step.soft_fail or false }}
|
23 |
-
retry:
|
24 |
-
automatic:
|
25 |
-
- exit_status: -1 # Agent was lost
|
26 |
-
limit: 5
|
27 |
-
plugins:
|
28 |
-
- kubernetes:
|
29 |
-
podSpec:
|
30 |
-
volumes:
|
31 |
-
- name: dshm
|
32 |
-
emptyDir:
|
33 |
-
medium: Memory
|
34 |
-
containers:
|
35 |
-
- image: "{{ docker_image }}"
|
36 |
-
command: ["bash"]
|
37 |
-
args:
|
38 |
-
- "-c"
|
39 |
-
- "'cd {{ (step.working_dir or default_working_dir) | safe }} && {{ step.command or (step.commands | join(' && ')) | safe }}'"
|
40 |
-
resources:
|
41 |
-
requests:
|
42 |
-
nvidia.com/gpu: "{{ step.num_gpus or default_num_gpu }}"
|
43 |
-
limits:
|
44 |
-
nvidia.com/gpu: "{{ step.num_gpus or default_num_gpu }}"
|
45 |
-
env:
|
46 |
-
- name: HF_TOKEN
|
47 |
-
valueFrom:
|
48 |
-
secretKeyRef:
|
49 |
-
name: hf-token-secret
|
50 |
-
key: token
|
51 |
-
volumeMounts:
|
52 |
-
- mountPath: /dev/shm
|
53 |
-
name: dshm
|
54 |
-
{% endfor %}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.dockerignore
DELETED
@@ -1 +0,0 @@
|
|
1 |
-
vllm/*.so
|
|
|
|
.gitattributes
DELETED
@@ -1,35 +0,0 @@
|
|
1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.github/workflows/publish.yml
DELETED
@@ -1,102 +0,0 @@
|
|
1 |
-
# This workflow will upload a Python Package to Release asset
|
2 |
-
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions
|
3 |
-
|
4 |
-
name: Create Release
|
5 |
-
|
6 |
-
on:
|
7 |
-
push:
|
8 |
-
tags:
|
9 |
-
- v*
|
10 |
-
|
11 |
-
# Needed to create release and upload assets
|
12 |
-
permissions:
|
13 |
-
contents: write
|
14 |
-
|
15 |
-
jobs:
|
16 |
-
release:
|
17 |
-
# Retrieve tag and create release
|
18 |
-
name: Create Release
|
19 |
-
runs-on: ubuntu-latest
|
20 |
-
outputs:
|
21 |
-
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
22 |
-
steps:
|
23 |
-
- name: Checkout
|
24 |
-
uses: actions/checkout@v3
|
25 |
-
|
26 |
-
- name: Extract branch info
|
27 |
-
shell: bash
|
28 |
-
run: |
|
29 |
-
echo "release_tag=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
|
30 |
-
|
31 |
-
- name: Create Release
|
32 |
-
id: create_release
|
33 |
-
uses: "actions/github-script@v6"
|
34 |
-
env:
|
35 |
-
RELEASE_TAG: ${{ env.release_tag }}
|
36 |
-
with:
|
37 |
-
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
38 |
-
script: |
|
39 |
-
const script = require('.github/workflows/scripts/create_release.js')
|
40 |
-
await script(github, context, core)
|
41 |
-
|
42 |
-
wheel:
|
43 |
-
name: Build Wheel
|
44 |
-
runs-on: ${{ matrix.os }}
|
45 |
-
needs: release
|
46 |
-
|
47 |
-
strategy:
|
48 |
-
fail-fast: false
|
49 |
-
matrix:
|
50 |
-
os: ['ubuntu-20.04']
|
51 |
-
python-version: ['3.8', '3.9', '3.10', '3.11']
|
52 |
-
pytorch-version: ['2.1.2'] # Must be the most recent version that meets requirements.txt.
|
53 |
-
cuda-version: ['11.8', '12.1']
|
54 |
-
|
55 |
-
steps:
|
56 |
-
- name: Checkout
|
57 |
-
uses: actions/checkout@v3
|
58 |
-
|
59 |
-
- name: Set up Linux Env
|
60 |
-
if: ${{ runner.os == 'Linux' }}
|
61 |
-
run: |
|
62 |
-
bash -x .github/workflows/scripts/env.sh
|
63 |
-
|
64 |
-
- name: Set up Python
|
65 |
-
uses: actions/setup-python@v4
|
66 |
-
with:
|
67 |
-
python-version: ${{ matrix.python-version }}
|
68 |
-
|
69 |
-
- name: Install CUDA ${{ matrix.cuda-version }}
|
70 |
-
run: |
|
71 |
-
bash -x .github/workflows/scripts/cuda-install.sh ${{ matrix.cuda-version }} ${{ matrix.os }}
|
72 |
-
|
73 |
-
- name: Install PyTorch ${{ matrix.pytorch-version }} with CUDA ${{ matrix.cuda-version }}
|
74 |
-
run: |
|
75 |
-
bash -x .github/workflows/scripts/pytorch-install.sh ${{ matrix.python-version }} ${{ matrix.pytorch-version }} ${{ matrix.cuda-version }}
|
76 |
-
|
77 |
-
- name: Build wheel
|
78 |
-
shell: bash
|
79 |
-
run: |
|
80 |
-
bash -x .github/workflows/scripts/build.sh ${{ matrix.python-version }} ${{ matrix.cuda-version }}
|
81 |
-
wheel_name=$(ls dist/*whl | xargs -n 1 basename)
|
82 |
-
asset_name=${wheel_name//"linux"/"manylinux1"}
|
83 |
-
echo "wheel_name=${wheel_name}" >> $GITHUB_ENV
|
84 |
-
echo "asset_name=${asset_name}" >> $GITHUB_ENV
|
85 |
-
|
86 |
-
- name: Upload Release Asset
|
87 |
-
uses: actions/upload-release-asset@v1
|
88 |
-
env:
|
89 |
-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
90 |
-
with:
|
91 |
-
upload_url: ${{ needs.release.outputs.upload_url }}
|
92 |
-
asset_path: ./dist/${{ env.wheel_name }}
|
93 |
-
asset_name: ${{ env.asset_name }}
|
94 |
-
asset_content_type: application/*
|
95 |
-
|
96 |
-
# (Danielkinz): This last step will publish the .whl to pypi. Warning: untested
|
97 |
-
# - name: Publish package
|
98 |
-
# uses: pypa/gh-action-pypi-publish@release/v1.8
|
99 |
-
# with:
|
100 |
-
# repository-url: https://test.pypi.org/legacy/
|
101 |
-
# password: ${{ secrets.PYPI_API_TOKEN }}
|
102 |
-
# skip-existing: true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.github/workflows/ruff.yml
DELETED
@@ -1,31 +0,0 @@
|
|
1 |
-
name: ruff
|
2 |
-
|
3 |
-
on:
|
4 |
-
# Trigger the workflow on push or pull request,
|
5 |
-
# but only for the main branch
|
6 |
-
push:
|
7 |
-
branches:
|
8 |
-
- main
|
9 |
-
pull_request:
|
10 |
-
branches:
|
11 |
-
- main
|
12 |
-
|
13 |
-
jobs:
|
14 |
-
ruff:
|
15 |
-
runs-on: ubuntu-latest
|
16 |
-
strategy:
|
17 |
-
matrix:
|
18 |
-
python-version: ["3.10"]
|
19 |
-
steps:
|
20 |
-
- uses: actions/checkout@v2
|
21 |
-
- name: Set up Python ${{ matrix.python-version }}
|
22 |
-
uses: actions/setup-python@v2
|
23 |
-
with:
|
24 |
-
python-version: ${{ matrix.python-version }}
|
25 |
-
- name: Install dependencies
|
26 |
-
run: |
|
27 |
-
python -m pip install --upgrade pip
|
28 |
-
pip install ruff==0.1.5
|
29 |
-
- name: Analysing the code with ruff
|
30 |
-
run: |
|
31 |
-
ruff vllm tests
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.github/workflows/scripts/build.sh
DELETED
@@ -1,20 +0,0 @@
|
|
1 |
-
#!/bin/bash
|
2 |
-
|
3 |
-
python_executable=python$1
|
4 |
-
cuda_home=/usr/local/cuda-$2
|
5 |
-
|
6 |
-
# Update paths
|
7 |
-
PATH=${cuda_home}/bin:$PATH
|
8 |
-
LD_LIBRARY_PATH=${cuda_home}/lib64:$LD_LIBRARY_PATH
|
9 |
-
|
10 |
-
# Install requirements
|
11 |
-
$python_executable -m pip install wheel packaging
|
12 |
-
$python_executable -m pip install -r requirements.txt
|
13 |
-
|
14 |
-
# Limit the number of parallel jobs to avoid OOM
|
15 |
-
export MAX_JOBS=1
|
16 |
-
# Make sure punica is built for the release (for LoRA)
|
17 |
-
export VLLM_INSTALL_PUNICA_KERNELS=1
|
18 |
-
|
19 |
-
# Build
|
20 |
-
$python_executable setup.py bdist_wheel --dist-dir=dist
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.github/workflows/scripts/create_release.js
DELETED
@@ -1,20 +0,0 @@
|
|
1 |
-
// Uses Github's API to create the release and wait for result.
|
2 |
-
// We use a JS script since github CLI doesn't provide a way to wait for the release's creation and returns immediately.
|
3 |
-
|
4 |
-
module.exports = async (github, context, core) => {
|
5 |
-
try {
|
6 |
-
const response = await github.rest.repos.createRelease({
|
7 |
-
draft: false,
|
8 |
-
generate_release_notes: true,
|
9 |
-
name: process.env.RELEASE_TAG,
|
10 |
-
owner: context.repo.owner,
|
11 |
-
prerelease: false,
|
12 |
-
repo: context.repo.repo,
|
13 |
-
tag_name: process.env.RELEASE_TAG,
|
14 |
-
});
|
15 |
-
|
16 |
-
core.setOutput('upload_url', response.data.upload_url);
|
17 |
-
} catch (error) {
|
18 |
-
core.setFailed(error.message);
|
19 |
-
}
|
20 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.github/workflows/scripts/cuda-install.sh
DELETED
@@ -1,23 +0,0 @@
|
|
1 |
-
#!/bin/bash
|
2 |
-
|
3 |
-
# Replace '.' with '-' ex: 11.8 -> 11-8
|
4 |
-
cuda_version=$(echo $1 | tr "." "-")
|
5 |
-
# Removes '-' and '.' ex: ubuntu-20.04 -> ubuntu2004
|
6 |
-
OS=$(echo $2 | tr -d ".\-")
|
7 |
-
|
8 |
-
# Installs CUDA
|
9 |
-
wget -nv https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-keyring_1.1-1_all.deb
|
10 |
-
sudo dpkg -i cuda-keyring_1.1-1_all.deb
|
11 |
-
rm cuda-keyring_1.1-1_all.deb
|
12 |
-
sudo apt -qq update
|
13 |
-
sudo apt -y install cuda-${cuda_version} cuda-nvcc-${cuda_version} cuda-libraries-dev-${cuda_version}
|
14 |
-
sudo apt clean
|
15 |
-
|
16 |
-
# Test nvcc
|
17 |
-
PATH=/usr/local/cuda-$1/bin:${PATH}
|
18 |
-
nvcc --version
|
19 |
-
|
20 |
-
# Log gcc, g++, c++ versions
|
21 |
-
gcc --version
|
22 |
-
g++ --version
|
23 |
-
c++ --version
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.github/workflows/scripts/env.sh
DELETED
@@ -1,56 +0,0 @@
|
|
1 |
-
#!/bin/bash
|
2 |
-
|
3 |
-
# This file installs common linux environment tools
|
4 |
-
|
5 |
-
export LANG C.UTF-8
|
6 |
-
|
7 |
-
# python_version=$1
|
8 |
-
|
9 |
-
sudo apt-get update && \
|
10 |
-
sudo apt-get install -y --no-install-recommends \
|
11 |
-
software-properties-common \
|
12 |
-
|
13 |
-
sudo apt-get install -y --no-install-recommends \
|
14 |
-
build-essential \
|
15 |
-
apt-utils \
|
16 |
-
ca-certificates \
|
17 |
-
wget \
|
18 |
-
git \
|
19 |
-
vim \
|
20 |
-
libssl-dev \
|
21 |
-
curl \
|
22 |
-
unzip \
|
23 |
-
unrar \
|
24 |
-
cmake \
|
25 |
-
net-tools \
|
26 |
-
sudo \
|
27 |
-
autotools-dev \
|
28 |
-
rsync \
|
29 |
-
jq \
|
30 |
-
openssh-server \
|
31 |
-
tmux \
|
32 |
-
screen \
|
33 |
-
htop \
|
34 |
-
pdsh \
|
35 |
-
openssh-client \
|
36 |
-
lshw \
|
37 |
-
dmidecode \
|
38 |
-
util-linux \
|
39 |
-
automake \
|
40 |
-
autoconf \
|
41 |
-
libtool \
|
42 |
-
net-tools \
|
43 |
-
pciutils \
|
44 |
-
libpci-dev \
|
45 |
-
libaio-dev \
|
46 |
-
libcap2 \
|
47 |
-
libtinfo5 \
|
48 |
-
fakeroot \
|
49 |
-
devscripts \
|
50 |
-
debhelper \
|
51 |
-
nfs-common
|
52 |
-
|
53 |
-
# Remove github bloat files to free up disk space
|
54 |
-
sudo rm -rf "/usr/local/share/boost"
|
55 |
-
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
56 |
-
sudo rm -rf "/usr/share/dotnet"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.github/workflows/scripts/pytorch-install.sh
DELETED
@@ -1,15 +0,0 @@
|
|
1 |
-
#!/bin/bash
|
2 |
-
|
3 |
-
python_executable=python$1
|
4 |
-
pytorch_version=$2
|
5 |
-
cuda_version=$3
|
6 |
-
|
7 |
-
# Install torch
|
8 |
-
$python_executable -m pip install numpy pyyaml scipy ipython mkl mkl-include ninja cython typing pandas typing-extensions dataclasses setuptools && conda clean -ya
|
9 |
-
$python_executable -m pip install torch==${pytorch_version}+cu${cuda_version//./} --extra-index-url https://download.pytorch.org/whl/cu${cuda_version//./}
|
10 |
-
|
11 |
-
# Print version information
|
12 |
-
$python_executable --version
|
13 |
-
$python_executable -c "import torch; print('PyTorch:', torch.__version__)"
|
14 |
-
$python_executable -c "import torch; print('CUDA:', torch.version.cuda)"
|
15 |
-
$python_executable -c "from torch.utils import cpp_extension; print (cpp_extension.CUDA_HOME)"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.github/workflows/yapf.yml
DELETED
@@ -1,31 +0,0 @@
|
|
1 |
-
name: yapf
|
2 |
-
|
3 |
-
on:
|
4 |
-
# Trigger the workflow on push or pull request,
|
5 |
-
# but only for the main branch
|
6 |
-
push:
|
7 |
-
branches:
|
8 |
-
- main
|
9 |
-
pull_request:
|
10 |
-
branches:
|
11 |
-
- main
|
12 |
-
jobs:
|
13 |
-
yapf:
|
14 |
-
runs-on: ubuntu-latest
|
15 |
-
strategy:
|
16 |
-
matrix:
|
17 |
-
python-version: ["3.10"]
|
18 |
-
steps:
|
19 |
-
- uses: actions/checkout@v2
|
20 |
-
- name: Set up Python ${{ matrix.python-version }}
|
21 |
-
uses: actions/setup-python@v2
|
22 |
-
with:
|
23 |
-
python-version: ${{ matrix.python-version }}
|
24 |
-
- name: Install dependencies
|
25 |
-
run: |
|
26 |
-
python -m pip install --upgrade pip
|
27 |
-
pip install yapf==0.32.0
|
28 |
-
pip install toml==0.10.2
|
29 |
-
- name: Running yapf
|
30 |
-
run: |
|
31 |
-
yapf --diff --recursive .
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.readthedocs.yaml
DELETED
@@ -1,21 +0,0 @@
|
|
1 |
-
# Read the Docs configuration file
|
2 |
-
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
|
3 |
-
|
4 |
-
version: 2
|
5 |
-
|
6 |
-
build:
|
7 |
-
os: ubuntu-22.04
|
8 |
-
tools:
|
9 |
-
python: "3.8"
|
10 |
-
|
11 |
-
sphinx:
|
12 |
-
configuration: docs/source/conf.py
|
13 |
-
|
14 |
-
# If using Sphinx, optionally build your docs in additional formats such as PDF
|
15 |
-
formats:
|
16 |
-
- pdf
|
17 |
-
|
18 |
-
# Optionally declare the Python requirements required to build your docs
|
19 |
-
python:
|
20 |
-
install:
|
21 |
-
- requirements: docs/requirements-docs.txt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
aiproxy/__main__.py
CHANGED
@@ -13,9 +13,9 @@ env_openai_api_key = "test"
|
|
13 |
|
14 |
# Get arguments
|
15 |
parser = argparse.ArgumentParser(description="UnaProxy usage")
|
16 |
-
parser.add_argument("--host", type=str, default=
|
17 |
-
parser.add_argument("--port", type=int, default=
|
18 |
-
parser.add_argument("--base_url", type=str, default=
|
19 |
parser.add_argument("--openai_api_key", type=str, default=env_openai_api_key, required=False, help="OpenAI API Key")
|
20 |
args = parser.parse_args()
|
21 |
|
|
|
13 |
|
14 |
# Get arguments
|
15 |
parser = argparse.ArgumentParser(description="UnaProxy usage")
|
16 |
+
parser.add_argument("--host", type=str, default=None, required=True, help="hostname or ipaddress")
|
17 |
+
parser.add_argument("--port", type=int, default=None, required=True, help="port number")
|
18 |
+
parser.add_argument("--base_url", type=str, default=None, required=True, help="port number")
|
19 |
parser.add_argument("--openai_api_key", type=str, default=env_openai_api_key, required=False, help="OpenAI API Key")
|
20 |
args = parser.parse_args()
|
21 |
|
aiproxy/aiproxy.db
DELETED
Binary file (94.2 kB)
|
|
requirements.txt
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
openai==1.3.6
|
2 |
+
fastapi==0.103.2
|
3 |
+
sse-starlette==1.8.2
|
4 |
+
uvicorn==0.23.2
|
5 |
+
tiktoken==0.5.1
|
6 |
+
SQLAlchemy==2.0.23
|
server.csr
DELETED
@@ -1,18 +0,0 @@
|
|
1 |
-
-----BEGIN CERTIFICATE REQUEST-----
|
2 |
-
MIICzzCCAbcCAQAwgYkxCzAJBgNVBAYTAlVTMRcwFQYDVQQIDA5Tb3V0aCBDYXJv
|
3 |
-
bGluYTETMBEGA1UEBwwKQ2hhcmxlc3RvbjERMA8GA1UECgwIUGVyc29uYWwxFTAT
|
4 |
-
BgNVBAMMDEFQSSBFbmRwb2ludDEiMCAGCSqGSIb3DQEJARYTYnNtaXQxNjU5QGdt
|
5 |
-
YWlsLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALvBXOWhiijs
|
6 |
-
Cwr9RuhBLv9zOu3U06j+PDBooKavhzKkKmUneOsufQdSfVLpul8zToWk9htngD2Z
|
7 |
-
zfyTsRjTg6wYMFXh//bxpqp2T5bYjH99lrhukCiN5grDmSSBjT4RFQUpt72MT6Uh
|
8 |
-
Dm+I17F/f7zwPG4l795CQyUaDL6yQzZmQIGrwKDsFkyUJXie6HR6rNRXakJwTU4w
|
9 |
-
asrNrVx94BuaCWWnNNfTmFwCrRGCO8mmSXU/hGTL7e5Po82JCjFdvVpw5ZSkP0oe
|
10 |
-
aS+KnAEYPkVqWUGP2Q++pA6DXe6OkVflGJBBL4sftIxoCMBHpSen/NbNS9/tPBaa
|
11 |
-
fYvqZTHMkkMCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4IBAQA7CaqC3ydIaSnVebz7
|
12 |
-
ciDv6RVgYJ5gxDp+BRk5Cbfoevs+ChP8zFOStsGL8jbk1VrNtGJZzC0FCHbMtuY3
|
13 |
-
lEg1+ZMpi8P2SMkMPjCSc5PbxB0Xp4y5ACyqVb/UMSI0r3ANbSirTkxysqxFSYd7
|
14 |
-
+xD644OVpCdVDC2JMp21P0EV/Qq8MZg8ZiUlO/FWW1+1OsHQ+gqG9vMofoNxbsM2
|
15 |
-
uKm7J1vmK/4n4raXb9BtSWoPUxVgag8giv2zkaV12ZKWsDP2wQzGi5qjrsJ62Fpt
|
16 |
-
YuPibZceaRufo4ntvYPYc5bgIfkXbJSJ1Fp9xoiZG49sU2dJhGJuWFkdlosmUBCz
|
17 |
-
xSoW
|
18 |
-
-----END CERTIFICATE REQUEST-----
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
server.key
DELETED
@@ -1,30 +0,0 @@
|
|
1 |
-
-----BEGIN RSA PRIVATE KEY-----
|
2 |
-
Proc-Type: 4,ENCRYPTED
|
3 |
-
DEK-Info: DES-EDE3-CBC,C7A79FD3DBB72768
|
4 |
-
|
5 |
-
cnLPOETFSBIO+bkqlHyINtceS9JYnqB2kNHHZv/hgMad76jaVH7g3IcUPBuA/5jP
|
6 |
-
jxILJy+2n1YYm/EWy6tG1CznScIs0CgixmAwC8+ql9OQSAa/3KqECcLSKTUso/VN
|
7 |
-
b9rINVgXtxyUmSA32rS8mqMW75U7grB0SpKvFoPn0EDPff+ohjFmApAPW1XPdKl+
|
8 |
-
rFwxhoMGFiR2p1mEjAmA6rXuKy+rwyQ4WjjNZQvPQZJsUMDF7SCy4xMIKUwns5op
|
9 |
-
6zc+DrgurovkZthXsbnvMd23onlEQtjFw0p25Ri4sg3YB/qbShGvouydoCcXRFKI
|
10 |
-
R5fjW1J+jU6g6sQSzm68ss0pwqYc1WYfKzhDbkrgjLN+0t+pYyxO3N2VmOSoweFl
|
11 |
-
TyH60yodjzBFDWWjxEMk7EjvhOcoIM+1pk2sXtsynMpjEFmKsZ0S3KB2x+0tvHMB
|
12 |
-
Qco2tYe/4VwkF7kIr7NHuVxFgIHLefTFgpER2yvOXrumSsjn6+bnwNjk9beasL9/
|
13 |
-
FghW6Vuo/8I6i+RyNLJBUE+ngsTuOu9ueTPqEex1b7hj9N7dLvZsBYTq3PdAYiB0
|
14 |
-
/e0zE07dzwv0lANkUYQEAqPF7LBek9Ff+pxB3yglIvVbW6QZ4Pv0gwfJzx9FQafR
|
15 |
-
LOwnmJad3xDPc2OHFbyNgpYq7mI9EGlfSEU+S0l0qClV0Zj9iL6H4whc36/FRNnm
|
16 |
-
3Zu8yzjEkhvS7yQqgOqIZ4z2/B/w5ucuxtSWt0iI738rUGXFlXFmTEyYyL0MgmWk
|
17 |
-
WvoF16BcyHOZO8UjRxtucTYe0kP93ZpVp09jkG0cow56hrRAuBlxauvZo1nlCSDO
|
18 |
-
MWqHCxEdWZOfQ14tosXqxkmHWf92QTNUZmatWFgeeqykfAWaIO08xaHZttrjK5kw
|
19 |
-
3oRe/uvO4raca6FzLbARxYy24T5dVxHm0STwdRIsiAZ1wz3Z49ZWfekKdXk/fbBK
|
20 |
-
DCn1yNxPM+KNdtS2tHKrSkpCSsUIzDBGViZPiSoH0imxfVkALqQ7jKvSTsn0jSW5
|
21 |
-
2EQn52OzURGq0IuD0Y/uIAMeblnSTsIi/exNfbSx9ryaL8T5GSUVCg6WIkJHgwXq
|
22 |
-
CCQMGsb76XEMoko8ShE34kLWg9MePDU0xg/esDlT7qcGmeXrijjaWmmV/L+1SMSZ
|
23 |
-
2pGhj2fyBSTbR2pdkjJE3O+5y+3jMjrDWjVWMWCzHMA7Ctgrpi70wiYEW07TPEno
|
24 |
-
LtvwiOuYdjBN7coO6cdSBQkh6ZIeqyvj9/z2mNJ8T7dKMNA0cI/H46MUwmYexe61
|
25 |
-
3iB5SbacJ8XsBEPXpUp2tmLgNEFOeYlQ9DfrH5uJ5mRPm++wgMR27gO8o3m+bMwD
|
26 |
-
y3adTZQaBQo4vuYH/Zq6JKLbMAmBG1k5xyh5i9+z7YFOAXmBQMLHMPJXVXAY8iRk
|
27 |
-
JKbaWufXHR+qnUxTfz3Cx09FayUFAbjyIehsV21o938tYkQ4rTwKSPKGPl2+ppQr
|
28 |
-
r6aa+FOJTw6jIGutK+NFBCUeKf75h/jMBsQgvFDa/vuaS5vh3AqaSK/C+JvnTpCJ
|
29 |
-
XmLN4WLWP+jSm9sTJQOzMb5aQrsopNx9nqrVVfA/Z3p+Rt6UNN6MeQ==
|
30 |
-
-----END RSA PRIVATE KEY-----
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|