import React, { useState, useEffect, useRef } from "react"; import { Card, Title, Subtitle, Table, TableHead, TableRow, TableHeaderCell, TableCell, TableBody, Metric, Text, Grid, Accordion, AccordionHeader, AccordionBody, } from "@tremor/react"; import { CredentialItem, credentialListCall, CredentialsResponse } from "./networking"; import ConditionalPublicModelName from "./add_model/conditional_public_model_name"; import LiteLLMModelNameField from "./add_model/litellm_model_name"; import AdvancedSettings from "./add_model/advanced_settings"; import ProviderSpecificFields from "./add_model/provider_specific_fields"; import { handleAddModelSubmit } from "./add_model/handle_add_model_submit"; import CredentialsPanel from "@/components/model_add/credentials"; import { getDisplayModelName } from "./view_model/model_name_display"; import EditModelModal, { handleEditModelSubmit } from "./edit_model/edit_model_modal"; import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon, DateRangePicker, } from "@tremor/react"; import { Select, SelectItem, MultiSelect, MultiSelectItem, DateRangePickerValue, } from "@tremor/react"; import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, streamingModelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall, getCallbacksCall, setCallbacksCall, modelSettingsCall, adminGlobalActivityExceptions, adminGlobalActivityExceptionsPerDeployment, allEndUsersCall, } from "./networking"; import { BarChart, AreaChart } from "@tremor/react"; import { Button as Button2, Modal, Popover, Form, Input, Select as AntdSelect, InputNumber, message, Descriptions, Tooltip, Space, Row, Col, } from "antd"; import { Badge, BadgeDelta, Button } from "@tremor/react"; import RequestAccess from "./request_model_access"; import { Typography } from "antd"; import TextArea from "antd/es/input/TextArea"; import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon, RefreshIcon, CheckCircleIcon, XCircleIcon, FilterIcon, ChevronUpIcon, ChevronDownIcon, TableIcon, } from "@heroicons/react/outline"; import DeleteModelButton from "./delete_model_button"; const { Title: Title2, Link } = Typography; import { UploadOutlined } from "@ant-design/icons"; import type { UploadProps } from "antd"; import { Upload } from "antd"; import TimeToFirstToken from "./model_metrics/time_to_first_token"; import DynamicFields from "./model_add/dynamic_form"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { Team } from "./key_team_helpers/key_list"; import TeamInfoView from "./team/team_info"; import { Providers, provider_map, providerLogoMap, getProviderLogoAndName, getPlaceholder, getProviderModels } from "./provider_info_helpers"; import ModelInfoView from "./model_info_view"; import AddModelTab from "./add_model/add_model_tab"; import { ModelDataTable } from "./model_dashboard/table"; import { columns } from "./model_dashboard/columns"; import { all_admin_roles } from "@/utils/roles"; import { Table as TableInstance } from '@tanstack/react-table'; interface ModelDashboardProps { accessToken: string | null; token: string | null; userRole: string | null; userID: string | null; modelData: any; keys: any[] | null; setModelData: any; premiumUser: boolean; teams: Team[] | null; } interface EditModelModalProps { visible: boolean; onCancel: () => void; model: any; // Assuming TeamType is a type representing your team object onSubmit: (data: FormData) => void; // Assuming FormData is the type of data to be submitted } interface RetryPolicyObject { [key: string]: { [retryPolicyKey: string]: number } | undefined; } interface GlobalExceptionActivityData { sum_num_rate_limit_exceptions: number; daily_data: { date: string; num_rate_limit_exceptions: number; }[]; } //["OpenAI", "Azure OpenAI", "Anthropic", "Gemini (Google AI Studio)", "Amazon Bedrock", "OpenAI-Compatible Endpoints (Groq, Together AI, Mistral AI, etc.)"] interface ProviderFields { field_name: string; field_type: string; field_description: string; field_value: string; } interface ProviderSettings { name: string; fields: ProviderFields[]; } const retry_policy_map: Record = { "BadRequestError (400)": "BadRequestErrorRetries", "AuthenticationError (401)": "AuthenticationErrorRetries", "TimeoutError (408)": "TimeoutErrorRetries", "RateLimitError (429)": "RateLimitErrorRetries", "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries", "InternalServerError (500)": "InternalServerErrorRetries", }; const ModelDashboard: React.FC = ({ accessToken, token, userRole, userID, modelData = { data: [] }, keys, setModelData, premiumUser, teams, }) => { const [pendingRequests, setPendingRequests] = useState([]); const [form] = Form.useForm(); const [modelMap, setModelMap] = useState(null); const [lastRefreshed, setLastRefreshed] = useState(""); const [providerModels, setProviderModels] = useState>([]); // Explicitly typing providerModels as a string array const providers = Object.values(Providers).filter((key) => isNaN(Number(key)) ); const [providerSettings, setProviderSettings] = useState( [] ); const [selectedProvider, setSelectedProvider] = useState(Providers.OpenAI); const [healthCheckResponse, setHealthCheckResponse] = useState(""); const [editModalVisible, setEditModalVisible] = useState(false); const [selectedModel, setSelectedModel] = useState(null); const [availableModelGroups, setAvailableModelGroups] = useState< Array >([]); const [availableProviders, setavailableProviders] = useState< Array >([]); const [selectedModelGroup, setSelectedModelGroup] = useState( null ); const [modelLatencyMetrics, setModelLatencyMetrics] = useState([]); const [modelMetrics, setModelMetrics] = useState([]); const [modelMetricsCategories, setModelMetricsCategories] = useState( [] ); const [streamingModelMetrics, setStreamingModelMetrics] = useState([]); const [streamingModelMetricsCategories, setStreamingModelMetricsCategories] = useState([]); const [modelExceptions, setModelExceptions] = useState([]); const [allExceptions, setAllExceptions] = useState([]); const [failureTableData, setFailureTableData] = useState([]); const [slowResponsesData, setSlowResponsesData] = useState([]); const [dateValue, setDateValue] = useState({ from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), to: new Date(), }); const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState(null); const [defaultRetry, setDefaultRetry] = useState(0); const [globalExceptionData, setGlobalExceptionData] = useState({} as GlobalExceptionActivityData); const [globalExceptionPerDeployment, setGlobalExceptionPerDeployment] = useState([]); const [showAdvancedFilters, setShowAdvancedFilters] = useState(false); const [selectedAPIKey, setSelectedAPIKey] = useState(null); const [selectedCustomer, setSelectedCustomer] = useState(null); const [allEndUsers, setAllEndUsers] = useState([]); const [credentialsList, setCredentialsList] = useState([]); // Add state for advanced settings visibility const [showAdvancedSettings, setShowAdvancedSettings] = useState(false); // Add these state variables const [selectedModelId, setSelectedModelId] = useState(null); const [editModel, setEditModel] = useState(false); const [selectedTeamId, setSelectedTeamId] = useState(null); const [selectedTeam, setSelectedTeam] = useState(null); const [selectedTeamFilter, setSelectedTeamFilter] = useState(null); const [showColumnDropdown, setShowColumnDropdown] = useState(false); const [isDropdownOpen, setIsDropdownOpen] = useState(false); const dropdownRef = useRef(null); const tableRef = useRef>(null); const setProviderModelsFn = (provider: Providers) => { const _providerModels = getProviderModels(provider, modelMap); setProviderModels(_providerModels); console.log(`providerModels: ${_providerModels}`); }; const updateModelMetrics = async ( modelGroup: string | null, startTime: Date | undefined, endTime: Date | undefined, ) => { console.log("Updating model metrics for group:", modelGroup); if (!accessToken || !userID || !userRole || !startTime || !endTime) { return; } console.log( "inside updateModelMetrics - startTime:", startTime, "endTime:", endTime ); setSelectedModelGroup(modelGroup); let selected_token = selectedAPIKey?.token; if (selected_token === undefined) { selected_token = null; } let selected_customer = selectedCustomer; if (selected_customer === undefined) { selected_customer = null; } // make startTime and endTime to last hour of the day startTime.setHours(0); startTime.setMinutes(0); startTime.setSeconds(0); endTime.setHours(23); endTime.setMinutes(59); endTime.setSeconds(59); try { const modelMetricsResponse = await modelMetricsCall( accessToken, userID, userRole, modelGroup, startTime.toISOString(), endTime.toISOString(), selected_token, selected_customer ); console.log("Model metrics response:", modelMetricsResponse); // Assuming modelMetricsResponse now contains the metric data for the specified model group setModelMetrics(modelMetricsResponse.data); setModelMetricsCategories(modelMetricsResponse.all_api_bases); const streamingModelMetricsResponse = await streamingModelMetricsCall( accessToken, modelGroup, startTime.toISOString(), endTime.toISOString() ); // Assuming modelMetricsResponse now contains the metric data for the specified model group setStreamingModelMetrics(streamingModelMetricsResponse.data); setStreamingModelMetricsCategories( streamingModelMetricsResponse.all_api_bases ); const modelExceptionsResponse = await modelExceptionsCall( accessToken, userID, userRole, modelGroup, startTime.toISOString(), endTime.toISOString(), selected_token, selected_customer ); console.log("Model exceptions response:", modelExceptionsResponse); setModelExceptions(modelExceptionsResponse.data); setAllExceptions(modelExceptionsResponse.exception_types); const slowResponses = await modelMetricsSlowResponsesCall( accessToken, userID, userRole, modelGroup, startTime.toISOString(), endTime.toISOString(), selected_token, selected_customer ); console.log("slowResponses:", slowResponses); setSlowResponsesData(slowResponses); if (modelGroup) { const dailyExceptions = await adminGlobalActivityExceptions( accessToken, startTime?.toISOString().split('T')[0], endTime?.toISOString().split('T')[0], modelGroup, ); setGlobalExceptionData(dailyExceptions); const dailyExceptionsPerDeplyment = await adminGlobalActivityExceptionsPerDeployment( accessToken, startTime?.toISOString().split('T')[0], endTime?.toISOString().split('T')[0], modelGroup, ) setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment); } } catch (error) { console.error("Failed to fetch model metrics", error); } }; const fetchCredentials = async (accessToken: string) => { try { const response: CredentialsResponse = await credentialListCall(accessToken); console.log(`credentials: ${JSON.stringify(response)}`); setCredentialsList(response.credentials); } catch (error) { console.error('Error fetching credentials:', error); } }; useEffect(() => { updateModelMetrics( selectedModelGroup, dateValue.from, dateValue.to ); }, [selectedAPIKey, selectedCustomer, selectedTeam]); useEffect(() => { const handleClickOutside = (event: MouseEvent) => { if (dropdownRef.current && !dropdownRef.current.contains(event.target as Node)) { setIsDropdownOpen(false); } }; document.addEventListener('mousedown', handleClickOutside); return () => document.removeEventListener('mousedown', handleClickOutside); }, []); function formatCreatedAt(createdAt: string | null) { if (createdAt) { const date = new Date(createdAt); const options = { month: "long", day: "numeric", year: "numeric" }; return date.toLocaleDateString("en-US"); } return null; } const handleEditClick = (model: any) => { setSelectedModel(model); setEditModalVisible(true); }; const handleEditCancel = () => { setEditModalVisible(false); setSelectedModel(null); }; const uploadProps: UploadProps = { name: "file", accept: ".json", beforeUpload: (file) => { if (file.type === "application/json") { const reader = new FileReader(); reader.onload = (e) => { if (e.target) { const jsonStr = e.target.result as string; form.setFieldsValue({ vertex_credentials: jsonStr }); } }; reader.readAsText(file); } // Prevent upload return false; }, onChange(info) { if (info.file.status !== "uploading") { console.log(info.file, info.fileList); } if (info.file.status === "done") { message.success(`${info.file.name} file uploaded successfully`); } else if (info.file.status === "error") { message.error(`${info.file.name} file upload failed.`); } }, }; const handleRefreshClick = () => { // Update the 'lastRefreshed' state to the current date and time const currentDate = new Date(); setLastRefreshed(currentDate.toLocaleString()); }; const handleSaveRetrySettings = async () => { if (!accessToken) { console.error("Access token is missing"); return; } console.log("new modelGroupRetryPolicy:", modelGroupRetryPolicy); try { const payload = { router_settings: { model_group_retry_policy: modelGroupRetryPolicy, }, }; await setCallbacksCall(accessToken, payload); message.success("Retry settings saved successfully"); } catch (error) { console.error("Failed to save retry settings:", error); message.error("Failed to save retry settings"); } }; useEffect(() => { if (!accessToken || !token || !userRole || !userID) { return; } const fetchData = async () => { try { // Replace with your actual API call for model data const modelDataResponse = await modelInfoCall( accessToken, userID, userRole ); console.log("Model data response:", modelDataResponse.data); setModelData(modelDataResponse); const _providerSettings = await modelSettingsCall(accessToken); if (_providerSettings) { setProviderSettings(_providerSettings); } // loop through modelDataResponse and get all`model_name` values let all_model_groups: Set = new Set(); for (let i = 0; i < modelDataResponse.data.length; i++) { const model = modelDataResponse.data[i]; all_model_groups.add(model.model_name); } console.log("all_model_groups:", all_model_groups); let _array_model_groups = Array.from(all_model_groups); // sort _array_model_groups alphabetically _array_model_groups = _array_model_groups.sort(); setAvailableModelGroups(_array_model_groups); console.log("array_model_groups:", _array_model_groups); let _initial_model_group = "all"; if (_array_model_groups.length > 0) { // set selectedModelGroup to the last model group _initial_model_group = _array_model_groups[_array_model_groups.length - 1]; console.log("_initial_model_group:", _initial_model_group); //setSelectedModelGroup(_initial_model_group); } console.log("selectedModelGroup:", selectedModelGroup); const modelMetricsResponse = await modelMetricsCall( accessToken, userID, userRole, _initial_model_group, dateValue.from?.toISOString(), dateValue.to?.toISOString(), selectedAPIKey?.token, selectedCustomer ); console.log("Model metrics response:", modelMetricsResponse); // Sort by latency (avg_latency_per_token) setModelMetrics(modelMetricsResponse.data); setModelMetricsCategories(modelMetricsResponse.all_api_bases); const streamingModelMetricsResponse = await streamingModelMetricsCall( accessToken, _initial_model_group, dateValue.from?.toISOString(), dateValue.to?.toISOString() ); // Assuming modelMetricsResponse now contains the metric data for the specified model group setStreamingModelMetrics(streamingModelMetricsResponse.data); setStreamingModelMetricsCategories( streamingModelMetricsResponse.all_api_bases ); const modelExceptionsResponse = await modelExceptionsCall( accessToken, userID, userRole, _initial_model_group, dateValue.from?.toISOString(), dateValue.to?.toISOString(), selectedAPIKey?.token, selectedCustomer ); console.log("Model exceptions response:", modelExceptionsResponse); setModelExceptions(modelExceptionsResponse.data); setAllExceptions(modelExceptionsResponse.exception_types); const slowResponses = await modelMetricsSlowResponsesCall( accessToken, userID, userRole, _initial_model_group, dateValue.from?.toISOString(), dateValue.to?.toISOString(), selectedAPIKey?.token, selectedCustomer ); const dailyExceptions = await adminGlobalActivityExceptions( accessToken, dateValue.from?.toISOString().split('T')[0], dateValue.to?.toISOString().split('T')[0], _initial_model_group, ); setGlobalExceptionData(dailyExceptions); const dailyExceptionsPerDeplyment = await adminGlobalActivityExceptionsPerDeployment( accessToken, dateValue.from?.toISOString().split('T')[0], dateValue.to?.toISOString().split('T')[0], _initial_model_group, ) setGlobalExceptionPerDeployment(dailyExceptionsPerDeplyment); console.log("dailyExceptions:", dailyExceptions); console.log("dailyExceptionsPerDeplyment:", dailyExceptionsPerDeplyment); console.log("slowResponses:", slowResponses); setSlowResponsesData(slowResponses); let all_end_users_data = await allEndUsersCall(accessToken); setAllEndUsers(all_end_users_data?.end_users); const routerSettingsInfo = await getCallbacksCall( accessToken, userID, userRole ); let router_settings = routerSettingsInfo.router_settings; console.log("routerSettingsInfo:", router_settings); let model_group_retry_policy = router_settings.model_group_retry_policy; let default_retries = router_settings.num_retries; console.log("model_group_retry_policy:", model_group_retry_policy); console.log("default_retries:", default_retries); setModelGroupRetryPolicy(model_group_retry_policy); setDefaultRetry(default_retries); } catch (error) { console.error("There was an error fetching the model data", error); } }; if (accessToken && token && userRole && userID) { fetchData(); } const fetchModelMap = async () => { const data = await modelCostMap(accessToken); console.log(`received model cost map data: ${Object.keys(data)}`); setModelMap(data); }; if (modelMap == null) { fetchModelMap(); } handleRefreshClick(); }, [accessToken, token, userRole, userID, modelMap, lastRefreshed, selectedTeam]); if (!modelData) { return
Loading...
; } if (!accessToken || !token || !userRole || !userID) { return
Loading...
; } let all_models_on_proxy: any[] = []; let all_providers: string[] = []; // loop through model data and edit each row for (let i = 0; i < modelData.data.length; i++) { let curr_model = modelData.data[i]; let litellm_model_name = curr_model?.litellm_params?.model; let custom_llm_provider = curr_model?.litellm_params?.custom_llm_provider; let model_info = curr_model?.model_info; let defaultProvider = "openai"; let provider = ""; let input_cost = "Undefined"; let output_cost = "Undefined"; let max_tokens = "Undefined"; let max_input_tokens = "Undefined"; let cleanedLitellmParams = {}; const getProviderFromModel = (model: string) => { /** * Use model map * - check if model in model map * - return it's litellm_provider, if so */ console.log(`GET PROVIDER CALLED! - ${modelMap}`); if (modelMap !== null && modelMap !== undefined) { if (typeof modelMap == "object" && model in modelMap) { return modelMap[model]["litellm_provider"]; } } return "openai"; }; // Check if litellm_model_name is null or undefined if (litellm_model_name) { // Split litellm_model_name based on "/" let splitModel = litellm_model_name.split("/"); // Get the first element in the split let firstElement = splitModel[0]; // If there is only one element, default provider to openai provider = custom_llm_provider; if (!provider) { provider = splitModel.length === 1 ? getProviderFromModel(litellm_model_name) : firstElement; } } else { // litellm_model_name is null or undefined, default provider to openai provider = "-"; } if (model_info) { input_cost = model_info?.input_cost_per_token; output_cost = model_info?.output_cost_per_token; max_tokens = model_info?.max_tokens; max_input_tokens = model_info?.max_input_tokens; } if (curr_model?.litellm_params) { cleanedLitellmParams = Object.fromEntries( Object.entries(curr_model?.litellm_params).filter( ([key]) => key !== "model" && key !== "api_base" ) ); } modelData.data[i].provider = provider; modelData.data[i].input_cost = input_cost; modelData.data[i].output_cost = output_cost; modelData.data[i].litellm_model_name = litellm_model_name; all_providers.push(provider); // Convert Cost in terms of Cost per 1M tokens if (modelData.data[i].input_cost) { modelData.data[i].input_cost = ( Number(modelData.data[i].input_cost) * 1000000 ).toFixed(2); } if (modelData.data[i].output_cost) { modelData.data[i].output_cost = ( Number(modelData.data[i].output_cost) * 1000000 ).toFixed(2); } modelData.data[i].max_tokens = max_tokens; modelData.data[i].max_input_tokens = max_input_tokens; modelData.data[i].api_base = curr_model?.litellm_params?.api_base; modelData.data[i].cleanedLitellmParams = cleanedLitellmParams; all_models_on_proxy.push(curr_model.model_name); console.log(modelData.data[i]); } // when users click request access show pop up to allow them to request access if (userRole && userRole == "Admin Viewer") { const { Title, Paragraph } = Typography; return (
Access Denied Ask your proxy admin for access to view all models
); } const runHealthCheck = async () => { try { message.info("Running health check..."); setHealthCheckResponse(""); const response = await healthCheckCall(accessToken); setHealthCheckResponse(response); } catch (error) { console.error("Error running health check:", error); setHealthCheckResponse("Error running health check"); } }; const FilterByContent = (
Select API Key Name { premiumUser ? (
Select Customer Name Select Team
): (
{/* ... existing non-premium user content ... */} Select Team
) }
); const customTooltip = (props: any) => { const { payload, active } = props; if (!active || !payload) return null; // Extract the date from the first item in the payload array const date = payload[0]?.payload?.date; // Sort the payload array by category.value in descending order let sortedPayload = payload.sort((a: any, b: any) => b.value - a.value); // Only show the top 5, the 6th one should be called "X other categories" depending on how many categories were not shown if (sortedPayload.length > 5) { let remainingItems = sortedPayload.length - 5; sortedPayload = sortedPayload.slice(0, 5); sortedPayload.push({ dataKey: `${remainingItems} other deployments`, value: payload .slice(5) .reduce((acc: number, curr: any) => acc + curr.value, 0), color: "gray", }); } return (
{date && (

Date: {date}

)} {sortedPayload.map((category: any, idx: number) => { const roundedValue = parseFloat(category.value.toFixed(5)); const displayValue = roundedValue === 0 && category.value > 0 ? "<0.00001" : roundedValue.toFixed(5); return (

{category.dataKey}

{displayValue}

); })}
); }; const handleOk = () => { form .validateFields() .then((values) => { handleAddModelSubmit(values, accessToken, form, handleRefreshClick); }) .catch((error) => { console.error("Validation failed:", error); }); }; console.log(`selectedProvider: ${selectedProvider}`); console.log(`providerModels.length: ${providerModels.length}`); const providerKey = Object.keys(Providers).find( (key) => (Providers as { [index: string]: any })[key] === selectedProvider ); let dynamicProviderForm: ProviderSettings | undefined = undefined; if (providerKey && providerSettings) { dynamicProviderForm = providerSettings.find( (provider) => provider.name === provider_map[providerKey] ); } // If a team is selected, render TeamInfoView in full page layout if (selectedTeamId) { return (
setSelectedTeamId(null)} accessToken={accessToken} is_team_admin={userRole === "Admin"} is_proxy_admin={userRole === "Proxy Admin"} userModels={all_models_on_proxy} editTeam={false} onUpdate={handleRefreshClick} />
); } return (
{selectedModelId ? ( { setSelectedModelId(null); setEditModel(false); }} modelData={modelData.data.find((model: any) => model.model_info.id === selectedModelId)} accessToken={accessToken} userID={userID} userRole={userRole} setEditModalVisible={setEditModalVisible} setSelectedModel={setSelectedModel} onModelUpdate={(updatedModel) => { // Update the model in the modelData.data array const updatedModelData = { ...modelData, data: modelData.data.map((model: any) => model.model_info.id === updatedModel.model_info.id ? updatedModel : model ) }; setModelData(updatedModelData); // Trigger a refresh to update UI handleRefreshClick(); }} /> ) : (
{all_admin_roles.includes(userRole) ? All Models : Your Models} Add Model {all_admin_roles.includes(userRole) && LLM Credentials} {all_admin_roles.includes(userRole) &&
/health Models
} {all_admin_roles.includes(userRole) && Model Analytics} {all_admin_roles.includes(userRole) && Model Retry Settings}
{lastRefreshed && Last Refreshed: {lastRefreshed}}
Model Management {!all_admin_roles.includes(userRole) ? ( Add models for teams you are an admin for. ) : ( Add and manage models for the proxy )}
{/* Search and Filter Controls */}
{/* Model Name Filter */}
Filter by Public Model Name:
{/* Team Filter */}
Filter by Team:
{/* Column Selector will be rendered here */} {/*
{isDropdownOpen && tableRef.current && (
{tableRef.current.getAllLeafColumns().map((column: any) => { if (column.id === 'actions') return null; return (
column.toggleVisibility()} > column.toggleVisibility()} className="h-4 w-4 rounded border-gray-300 text-blue-600 focus:ring-blue-500" /> {typeof column.columnDef.header === 'string' ? column.columnDef.header : column.columnDef.header?.props?.children || column.id}
); })}
)}
*/}
{/* Results Count */}
Showing {modelData && modelData.data.length > 0 ? modelData.data.length : 0} results
( (selectedModelGroup === "all" || model.model_name === selectedModelGroup || !selectedModelGroup) && (selectedTeamFilter === "all" || model.model_info["team_id"] === selectedTeamFilter || !selectedTeamFilter) ) )} isLoading={false} table={tableRef} />
`/health` will run a very small request through your models configured on litellm {healthCheckResponse && (
{JSON.stringify(healthCheckResponse, null, 2)}
)}
Select Time Range { setDateValue(value); updateModelMetrics( selectedModelGroup, value.from, value.to ); // Call updateModelMetrics with the new date range }} /> Select Model Group Avg. Latency per Token Time to first token

(seconds/token)

average Latency for successfull requests divided by the total tokens {modelMetrics && modelMetricsCategories && ( )}
Deployment Success Responses Slow Responses

Success Responses taking 600+s

{slowResponsesData.map((metric, idx) => ( {metric.api_base} {metric.total_count} {metric.slow_count} ))}
All Exceptions for {selectedModelGroup} All Up Rate Limit Errors (429) for {selectedModelGroup} Num Rate Limit Errors { (globalExceptionData.sum_num_rate_limit_exceptions)} console.log(v)} /> { premiumUser ? ( <> {globalExceptionPerDeployment.map((globalActivity, index) => ( {globalActivity.api_base ? globalActivity.api_base : "Unknown API Base"} Num Rate Limit Errors (429) {(globalActivity.sum_num_rate_limit_exceptions)} console.log(v)} /> ))} ) : <> {globalExceptionPerDeployment && globalExceptionPerDeployment.length > 0 && globalExceptionPerDeployment.slice(0, 1).map((globalActivity, index) => ( ✨ Rate Limit Errors by Deployment

Upgrade to see exceptions for all deployments

{globalActivity.api_base} Num Rate Limit Errors {(globalActivity.sum_num_rate_limit_exceptions)} console.log(v)} />
))} }
Filter by Public Model Name
Retry Policy for {selectedModelGroup} How many retries should be attempted based on the Exception {retry_policy_map && ( {Object.entries(retry_policy_map).map( ([exceptionType, retryPolicyKey], idx) => { let retryCount = modelGroupRetryPolicy?.[selectedModelGroup!]?.[ retryPolicyKey ]; if (retryCount == null) { retryCount = defaultRetry; } return ( ); } )}
{exceptionType} { setModelGroupRetryPolicy( (prevModelGroupRetryPolicy) => { const prevRetryPolicy = prevModelGroupRetryPolicy?.[ selectedModelGroup! ] ?? {}; return { ...(prevModelGroupRetryPolicy ?? {}), [selectedModelGroup!]: { ...prevRetryPolicy, [retryPolicyKey!]: value, }, } as RetryPolicyObject; } ); }} />
)}
)}
); }; export default ModelDashboard;