fda

proxy_server.py

@@ -7,6 +7,8 @@ import secrets, subprocess
 import hashlib, uuid
 import warnings
 import importlib
+from groq import Groq
+import os
 #######################
 messages: list = []
 sys.path.insert(
@@ -2480,6 +2482,24 @@ async def config_yaml_endpoint(config_info: ConfigYAML):
     """
     return {"hello": "world"}
 
+@router.get("/groq/chat", tags=["groq"])
+async def config_yaml_endpoint(config_info: ConfigYAML):
+    client = Groq(
+        api_key=os.environ.get("GROQ_API_KEY"),
+    )
+
+    chat_completion = client.chat.completions.create(
+        messages=[
+            {
+                "role": "user",
+                "content": "Explain the importance of fast language models",
+            }
+        ],
+        model="llama3-8b-8192",
+    )
+
+    print(chat_completion.choices[0].message.content)    
+
 
 @router.get("/test", tags=["health"])
 async def test_endpoint(request: Request):

requirements.txt

@@ -25,4 +25,5 @@ click==8.1.7 # for proxy cli
 jinja2==3.1.2 # for prompt templates
 certifi>=2023.7.22 # [TODO] clean up 
 aiohttp==3.9.0 # for network calls
+groq
 ####