File size: 2,788 Bytes
7db5a18
 
 
 
bc00a8c
c18033f
94bd789
7db5a18
 
 
3ed391d
7db5a18
 
 
 
 
bc00a8c
c18033f
94bd789
7db5a18
 
 
 
 
 
bc00a8c
c18033f
c66e20f
94bd789
7db5a18
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bc00a8c
 
7db5a18
 
bc00a8c
7db5a18
 
3ebae27
c18033f
7db5a18
 
 
 
94bd789
 
 
 
 
 
3ed391d
c66ed52
c66e20f
c66ed52
 
 
 
 
7db5a18
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
MODEL_MAP = {
    "mixtral-8x7b": "mistralai/Mixtral-8x7B-Instruct-v0.1",  # [Recommended]
    "nous-mixtral-8x7b": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
    "mistral-7b": "mistralai/Mistral-7B-Instruct-v0.2",
    "openchat-3.5": "openchat/openchat-3.5-0106",
    "gemma-1.1-7b": "google/gemma-1.1-7b-it",
    "command-r-plus": "CohereForAI/c4ai-command-r-plus",
    "default": "mistralai/Mixtral-8x7B-Instruct-v0.1",
}

AVAILABLE_MODELS = list(MODEL_MAP.keys())

STOP_SEQUENCES_MAP = {
    "mixtral-8x7b": "</s>",
    "nous-mixtral-8x7b": "<|im_end|>",
    "mistral-7b": "</s>",
    "openchat-3.5": "<|end_of_turn|>",
    "gemma-1.1-7b": "<eos>",
    "command-r-plus": "<|END_OF_TURN_TOKEN|>",
}

TOKEN_LIMIT_MAP = {
    "mixtral-8x7b": 32768,
    "nous-mixtral-8x7b": 32768,
    "mistral-7b": 32768,
    "openchat-3.5": 8192,
    "gemma-1.1-7b": 8192,
    "gpt-3.5-turbo": 8192,
    "command-r-plus": 32768,
}

TOKEN_RESERVED = 20


# https://platform.openai.com/docs/api-reference/models/list
AVAILABLE_MODELS_DICTS = [
    {
        "id": "mixtral-8x7b",
        "description": "[mistralai/Mixtral-8x7B-Instruct-v0.1]: https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1",
        "object": "model",
        "created": 1700000000,
        "owned_by": "mistralai",
    },
    {
        "id": "nous-mixtral-8x7b",
        "description": "[NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO]: https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
        "object": "model",
        "created": 1700000000,
        "owned_by": "NousResearch",
    },
    {
        "id": "mistral-7b",
        "description": "[mistralai/Mistral-7B-Instruct-v0.2]: https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2",
        "object": "model",
        "created": 1700000000,
        "owned_by": "mistralai",
    },
    {
        "id": "openchat-3.5",
        "description": "[openchat/openchat-3.5-0106]: https://huggingface.co/openchat/openchat-3.5-0106",
        "object": "model",
        "created": 1700000000,
        "owned_by": "openchat",
    },
    {
        "id": "gemma-1.1-7b",
        "description": "[google/gemma-1.1-7b-it]: https://huggingface.co/google/gemma-1.1-7b-it",
        "object": "model",
        "created": 1700000000,
        "owned_by": "Google",
    },
    {
        "id": "command-r-plus",
        "description": "[CohereForAI/c4ai-command-r-plus]: https://huggingface.co/CohereForAI/c4ai-command-r-plus",
        "object": "model",
        "created": 1700000000,
        "owned_by": "CohereForAI",
    },
    {
        "id": "gpt-3.5-turbo",
        "description": "[openai/gpt-3.5-turbo]: https://platform.openai.com/docs/models/gpt-3-5-turbo",
        "object": "model",
        "created": 1700000000,
        "owned_by": "OpenAI",
    },
]