Spaces:
Running
Running
File size: 2,233 Bytes
6ff1f88 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 |
from openai import AsyncOpenAI
import httpx
async def run_model_stream(api_key: str, model: str, prompt: str):
"""
Run the Groq model with streaming response.
Args:
api_key: The API key to use for this request
model: The model name to use
prompt: The user's input prompt
Yields:
str: Chunks of the generated response
"""
try:
# Initialize AsyncOpenAI client with specific configuration
client = AsyncOpenAI(
api_key=api_key,
base_url="https://api.groq.com/openai/v1",
http_client=httpx.AsyncClient(verify=True) # Async client
)
completion = await client.chat.completions.create(
messages=[
{
"role": "user",
"content": prompt
}
],
model=model,
stream=True
)
async for chunk in completion:
if chunk.choices[0].delta.content is not None:
yield chunk.choices[0].delta.content
except Exception as e:
raise Exception(f"Error with Groq API: {str(e)}")
async def run_model(api_key: str, model: str, prompt: str) -> str:
"""
Run the Groq model with the provided API key and prompt (non-streaming).
Args:
api_key: The API key to use for this request
model: The model name to use
prompt: The user's input prompt
Returns:
str: The generated response
"""
try:
# Initialize AsyncOpenAI client with specific configuration
client = AsyncOpenAI(
api_key=api_key,
base_url="https://api.groq.com/openai/v1",
http_client=httpx.AsyncClient(verify=True) # Async client
)
chat_completion = await client.chat.completions.create(
messages=[
{
"role": "user",
"content": prompt
}
],
model=model
)
return chat_completion.choices[0].message.content
except Exception as e:
raise Exception(f"Error with Groq API: {str(e)}")
|