LiYongHui yonghui li commited on
Commit
29b8637
·
1 Parent(s): 69e5f04

Integrates LLM Azure OpenAI (#1318)

Browse files

### What problem does this PR solve?

feat: Integrates LLM Azure OpenAI #716

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

### Other
It's just the back-end code, the front-end needs to provide the Azure
OpenAI model addition form.

#### Required parameters

- base_url
- api_key

---------

Co-authored-by: yonghui li <[email protected]>

api/db/init_data.py CHANGED
@@ -165,6 +165,11 @@ factory_infos = [{
165
  "logo": "",
166
  "tags": "LLM,TEXT EMBEDDING",
167
  "status": "1",
 
 
 
 
 
168
  }
169
  # {
170
  # "name": "文心一言",
@@ -649,6 +654,83 @@ def init_llm_factory():
649
  "max_tokens": 8192,
650
  "model_type": LLMType.EMBEDDING
651
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
652
  ]
653
  for info in factory_infos:
654
  try:
 
165
  "logo": "",
166
  "tags": "LLM,TEXT EMBEDDING",
167
  "status": "1",
168
+ },{
169
+ "name": "Azure-OpenAI",
170
+ "logo": "",
171
+ "tags": "LLM,TEXT EMBEDDING,SPEECH2TEXT,MODERATION",
172
+ "status": "1",
173
  }
174
  # {
175
  # "name": "文心一言",
 
654
  "max_tokens": 8192,
655
  "model_type": LLMType.EMBEDDING
656
  },
657
+ # ------------------------ Azure OpenAI -----------------------
658
+ # Please ensure the llm_name is the same as the name in Azure
659
+ # OpenAI deployment name (e.g., azure-gpt-4o). And the llm_name
660
+ # must different from the OpenAI llm_name
661
+ #
662
+ # Each model must be deployed in the Azure OpenAI service, otherwise,
663
+ # you will receive an error message 'The API deployment for
664
+ # this resource does not exist'
665
+ {
666
+ "fid": factory_infos[15]["name"],
667
+ "llm_name": "azure-gpt-4o",
668
+ "tags": "LLM,CHAT,128K",
669
+ "max_tokens": 128000,
670
+ "model_type": LLMType.CHAT.value + "," + LLMType.IMAGE2TEXT.value
671
+ }, {
672
+ "fid": factory_infos[15]["name"],
673
+ "llm_name": "azure-gpt-35-turbo",
674
+ "tags": "LLM,CHAT,4K",
675
+ "max_tokens": 4096,
676
+ "model_type": LLMType.CHAT.value
677
+ }, {
678
+ "fid": factory_infos[15]["name"],
679
+ "llm_name": "azure-gpt-35-turbo-16k",
680
+ "tags": "LLM,CHAT,16k",
681
+ "max_tokens": 16385,
682
+ "model_type": LLMType.CHAT.value
683
+ }, {
684
+ "fid": factory_infos[15]["name"],
685
+ "llm_name": "azure-text-embedding-ada-002",
686
+ "tags": "TEXT EMBEDDING,8K",
687
+ "max_tokens": 8191,
688
+ "model_type": LLMType.EMBEDDING.value
689
+ }, {
690
+ "fid": factory_infos[15]["name"],
691
+ "llm_name": "azure-text-embedding-3-small",
692
+ "tags": "TEXT EMBEDDING,8K",
693
+ "max_tokens": 8191,
694
+ "model_type": LLMType.EMBEDDING.value
695
+ }, {
696
+ "fid": factory_infos[15]["name"],
697
+ "llm_name": "azure-text-embedding-3-large",
698
+ "tags": "TEXT EMBEDDING,8K",
699
+ "max_tokens": 8191,
700
+ "model_type": LLMType.EMBEDDING.value
701
+ },{
702
+ "fid": factory_infos[15]["name"],
703
+ "llm_name": "azure-whisper-1",
704
+ "tags": "SPEECH2TEXT",
705
+ "max_tokens": 25 * 1024 * 1024,
706
+ "model_type": LLMType.SPEECH2TEXT.value
707
+ },
708
+ {
709
+ "fid": factory_infos[15]["name"],
710
+ "llm_name": "azure-gpt-4",
711
+ "tags": "LLM,CHAT,8K",
712
+ "max_tokens": 8191,
713
+ "model_type": LLMType.CHAT.value
714
+ }, {
715
+ "fid": factory_infos[15]["name"],
716
+ "llm_name": "azure-gpt-4-turbo",
717
+ "tags": "LLM,CHAT,8K",
718
+ "max_tokens": 8191,
719
+ "model_type": LLMType.CHAT.value
720
+ }, {
721
+ "fid": factory_infos[15]["name"],
722
+ "llm_name": "azure-gpt-4-32k",
723
+ "tags": "LLM,CHAT,32K",
724
+ "max_tokens": 32768,
725
+ "model_type": LLMType.CHAT.value
726
+ }, {
727
+ "fid": factory_infos[15]["name"],
728
+ "llm_name": "azure-gpt-4-vision-preview",
729
+ "tags": "LLM,CHAT,IMAGE2TEXT",
730
+ "max_tokens": 765,
731
+ "model_type": LLMType.IMAGE2TEXT.value
732
+ },
733
+
734
  ]
735
  for info in factory_infos:
736
  try:
api/settings.py CHANGED
@@ -69,6 +69,12 @@ default_llm = {
69
  "image2text_model": "gpt-4-vision-preview",
70
  "asr_model": "whisper-1",
71
  },
 
 
 
 
 
 
72
  "ZHIPU-AI": {
73
  "chat_model": "glm-3-turbo",
74
  "embedding_model": "embedding-2",
 
69
  "image2text_model": "gpt-4-vision-preview",
70
  "asr_model": "whisper-1",
71
  },
72
+ "Azure-OpenAI": {
73
+ "chat_model": "azure-gpt-35-turbo",
74
+ "embedding_model": "azure-text-embedding-ada-002",
75
+ "image2text_model": "azure-gpt-4-vision-preview",
76
+ "asr_model": "azure-whisper-1",
77
+ },
78
  "ZHIPU-AI": {
79
  "chat_model": "glm-3-turbo",
80
  "embedding_model": "embedding-2",
rag/llm/__init__.py CHANGED
@@ -22,6 +22,7 @@ from .rerank_model import *
22
  EmbeddingModel = {
23
  "Ollama": OllamaEmbed,
24
  "OpenAI": OpenAIEmbed,
 
25
  "Xinference": XinferenceEmbed,
26
  "Tongyi-Qianwen": QWenEmbed,
27
  "ZHIPU-AI": ZhipuEmbed,
@@ -36,6 +37,7 @@ EmbeddingModel = {
36
 
37
  CvModel = {
38
  "OpenAI": GptV4,
 
39
  "Ollama": OllamaCV,
40
  "Xinference": XinferenceCV,
41
  "Tongyi-Qianwen": QWenCV,
@@ -46,6 +48,7 @@ CvModel = {
46
 
47
  ChatModel = {
48
  "OpenAI": GptTurbo,
 
49
  "ZHIPU-AI": ZhipuChat,
50
  "Tongyi-Qianwen": QWenChat,
51
  "Ollama": OllamaChat,
 
22
  EmbeddingModel = {
23
  "Ollama": OllamaEmbed,
24
  "OpenAI": OpenAIEmbed,
25
+ "Azure-OpenAI": AzureEmbed,
26
  "Xinference": XinferenceEmbed,
27
  "Tongyi-Qianwen": QWenEmbed,
28
  "ZHIPU-AI": ZhipuEmbed,
 
37
 
38
  CvModel = {
39
  "OpenAI": GptV4,
40
+ "Azure-OpenAI": AzureGptV4,
41
  "Ollama": OllamaCV,
42
  "Xinference": XinferenceCV,
43
  "Tongyi-Qianwen": QWenCV,
 
48
 
49
  ChatModel = {
50
  "OpenAI": GptTurbo,
51
+ "Azure-OpenAI": AzureChat,
52
  "ZHIPU-AI": ZhipuChat,
53
  "Tongyi-Qianwen": QWenChat,
54
  "Ollama": OllamaChat,
rag/llm/chat_model.py CHANGED
@@ -13,6 +13,7 @@
13
  # See the License for the specific language governing permissions and
14
  # limitations under the License.
15
  #
 
16
  from zhipuai import ZhipuAI
17
  from dashscope import Generation
18
  from abc import ABC
@@ -94,6 +95,11 @@ class DeepSeekChat(Base):
94
  if not base_url: base_url="https://api.deepseek.com/v1"
95
  super().__init__(key, model_name, base_url)
96
 
 
 
 
 
 
97
 
98
  class BaiChuanChat(Base):
99
  def __init__(self, key, model_name="Baichuan3-Turbo", base_url="https://api.baichuan-ai.com/v1"):
 
13
  # See the License for the specific language governing permissions and
14
  # limitations under the License.
15
  #
16
+ from openai.lib.azure import AzureOpenAI
17
  from zhipuai import ZhipuAI
18
  from dashscope import Generation
19
  from abc import ABC
 
95
  if not base_url: base_url="https://api.deepseek.com/v1"
96
  super().__init__(key, model_name, base_url)
97
 
98
+ class AzureChat(Base):
99
+ def __init__(self, key, model_name, **kwargs):
100
+ self.client = AzureOpenAI(api_key=key, azure_endpoint=kwargs["base_url"], api_version="2024-02-01")
101
+ self.model_name = model_name
102
+
103
 
104
  class BaiChuanChat(Base):
105
  def __init__(self, key, model_name="Baichuan3-Turbo", base_url="https://api.baichuan-ai.com/v1"):
rag/llm/cv_model.py CHANGED
@@ -13,6 +13,7 @@
13
  # See the License for the specific language governing permissions and
14
  # limitations under the License.
15
  #
 
16
  from zhipuai import ZhipuAI
17
  import io
18
  from abc import ABC
@@ -87,6 +88,25 @@ class GptV4(Base):
87
  )
88
  return res.choices[0].message.content.strip(), res.usage.total_tokens
89
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
90
 
91
  class QWenCV(Base):
92
  def __init__(self, key, model_name="qwen-vl-chat-v1", lang="Chinese", **kwargs):
 
13
  # See the License for the specific language governing permissions and
14
  # limitations under the License.
15
  #
16
+ from openai.lib.azure import AzureOpenAI
17
  from zhipuai import ZhipuAI
18
  import io
19
  from abc import ABC
 
88
  )
89
  return res.choices[0].message.content.strip(), res.usage.total_tokens
90
 
91
+ class AzureGptV4(Base):
92
+ def __init__(self, key, model_name, lang="Chinese", **kwargs):
93
+ self.client = AzureOpenAI(api_key=key, azure_endpoint=kwargs["base_url"], api_version="2024-02-01")
94
+ self.model_name = model_name
95
+ self.lang = lang
96
+
97
+ def describe(self, image, max_tokens=300):
98
+ b64 = self.image2base64(image)
99
+ prompt = self.prompt(b64)
100
+ for i in range(len(prompt)):
101
+ for c in prompt[i]["content"]:
102
+ if "text" in c: c["type"] = "text"
103
+
104
+ res = self.client.chat.completions.create(
105
+ model=self.model_name,
106
+ messages=prompt,
107
+ max_tokens=max_tokens,
108
+ )
109
+ return res.choices[0].message.content.strip(), res.usage.total_tokens
110
 
111
  class QWenCV(Base):
112
  def __init__(self, key, model_name="qwen-vl-chat-v1", lang="Chinese", **kwargs):
rag/llm/embedding_model.py CHANGED
@@ -18,6 +18,7 @@ from typing import Optional
18
  import threading
19
  import requests
20
  from huggingface_hub import snapshot_download
 
21
  from zhipuai import ZhipuAI
22
  import os
23
  from abc import ABC
@@ -110,6 +111,11 @@ class OpenAIEmbed(Base):
110
  return np.array(res.data[0].embedding), res.usage.total_tokens
111
 
112
 
 
 
 
 
 
113
  class BaiChuanEmbed(OpenAIEmbed):
114
  def __init__(self, key,
115
  model_name='Baichuan-Text-Embedding',
 
18
  import threading
19
  import requests
20
  from huggingface_hub import snapshot_download
21
+ from openai.lib.azure import AzureOpenAI
22
  from zhipuai import ZhipuAI
23
  import os
24
  from abc import ABC
 
111
  return np.array(res.data[0].embedding), res.usage.total_tokens
112
 
113
 
114
+ class AzureEmbed(Base):
115
+ def __init__(self, key, model_name, **kwargs):
116
+ self.client = AzureOpenAI(api_key=key, azure_endpoint=kwargs["base_url"], api_version="2024-02-01")
117
+ self.model_name = model_name
118
+
119
  class BaiChuanEmbed(OpenAIEmbed):
120
  def __init__(self, key,
121
  model_name='Baichuan-Text-Embedding',