add snippets

src/lib/components/InferencePlayground/InferencePlayground.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import type { Conversation, ConversationMessage, ModelEntryWithTokenizer } from "./types";
+	import type { Conversation, ConversationMessage, ModelWithTokenizer } from "$lib/types";
 
 	import {
 		handleNonStreamingResponse,
@@ -194,7 +194,7 @@
 		}
 	}
 
-	function addCompareModel(modelId: ModelEntryWithTokenizer["id"]) {
+	function addCompareModel(modelId: ModelWithTokenizer["id"]) {
 		const model = $models.find(m => m.id === modelId);
 		if (!model || $session.conversations.length === 2) {
 			return;

src/lib/components/InferencePlayground/InferencePlaygroundCodeSnippets.svelte

@@ -1,15 +1,20 @@
 <script lang="ts">
-	import type { Conversation } from "./types";
+	import type { Conversation } from "$lib/types";
 
-	import { createEventDispatcher, onDestroy } from "svelte";
 	import hljs from "highlight.js/lib/core";
+	import http from "highlight.js/lib/languages/http";
 	import javascript from "highlight.js/lib/languages/javascript";
 	import python from "highlight.js/lib/languages/python";
-	import http from "highlight.js/lib/languages/http";
+	import { createEventDispatcher, onDestroy } from "svelte";
 
-	import IconCopyCode from "../Icons/IconCopyCode.svelte";
-	import { isSystemPromptSupported } from "./inferencePlaygroundUtils";
 	import { token } from "$lib/stores/token";
+	import type { InferenceProvider } from "@huggingface/inference";
+	import IconCopyCode from "../Icons/IconCopyCode.svelte";
+	import {
+		getInferenceSnippet,
+		type GetInferenceSnippetReturn,
+		type InferenceSnippetLanguage,
+	} from "./inferencePlaygroundUtils";
 
 	hljs.registerLanguage("javascript", javascript);
 	hljs.registerLanguage("python", python);
@@ -24,44 +29,26 @@
 	const labelsByLanguage: Record<Language, string> = {
 		javascript: "JavaScript",
 		python: "Python",
-		http: "Curl",
+		http: "cURL",
 	};
 
-	interface Snippet {
-		label: string;
-		code: string;
-		language?: Language;
-		needsToken?: boolean;
-	}
-
-	interface ClientSnippet {
-		name: string;
-		snippets: Snippet[];
-	}
-
-	interface MessagesJoiner {
-		sep: string;
-		start: string;
-		end: string;
-	}
-
 	let selectedLanguage: Language = "javascript";
 	let timeout: ReturnType<typeof setTimeout>;
 	let showToken = false;
 
 	$: tokenStr = getTokenStr(showToken);
 
+	function getSnippet(lang: InferenceSnippetLanguage, tokenStr: string) {
+		return getInferenceSnippet(conversation.model, conversation.provider as InferenceProvider, lang, tokenStr, {
+			messages: conversation.messages,
+		});
+	}
+
 	$: clientSnippetsByLang = {
-		javascript: [
-			{ name: "@huggingface/inference", snippets: getJavascriptSnippetsHfClient(conversation, tokenStr) },
-			{ name: "openai", snippets: getJavascriptSnippetsOAIClient(conversation, tokenStr) },
-		],
-		python: [
-			{ name: "huggingface_hub", snippets: getPythonSnippetsHfClient(conversation, tokenStr) },
-			{ name: "openai", snippets: getPythonSnippetsOAIClient(conversation, tokenStr) },
-		],
-		http: [{ name: "curl", snippets: getHttpSnippets(conversation, tokenStr) }],
-	} as Record<Language, ClientSnippet[]>;
+		javascript: getSnippet("js", tokenStr),
+		python: getSnippet("python", tokenStr),
+		http: getSnippet("curl", tokenStr),
+	} as Record<Language, GetInferenceSnippetReturn>;
 
 	const selectedClientIdxByLang: Record<Language, number> = Object.fromEntries(lanuages.map(lang => [lang, 0]));
 
@@ -72,370 +59,12 @@
 		return "YOUR_HF_TOKEN";
 	}
 
-	function getMessages() {
-		const placeholder = [{ role: "user", content: "Tell me a story" }];
-
-		let messages = [...conversation.messages];
-		if (messages.length === 1 && messages[0]?.role === "user" && !messages[0]?.content) {
-			messages = placeholder;
-		}
-
-		const { model, systemMessage } = conversation;
-		if (isSystemPromptSupported(model) && systemMessage.content?.length) {
-			messages.unshift(systemMessage);
-		}
-
-		const res = messages.map(({ role, content }) => ({
-			role,
-			content: JSON.stringify(content).slice(1, -1),
-		}));
-		messages = res;
-
-		return res;
-	}
-
 	function highlight(code: string, language: Language) {
-		return hljs.highlight(code, { language }).value;
-	}
-
-	function getJavascriptSnippetsHfClient(conversation: Conversation, tokenStr: string) {
-		const formattedMessages = ({ sep, start, end }: MessagesJoiner) =>
-			start +
-			getMessages()
-				.map(({ role, content }) => `{ role: "${role}", content: "${content}" }`)
-				.join(sep) +
-			end;
-
-		const formattedConfig = ({ sep, start, end }: MessagesJoiner) =>
-			start +
-			Object.entries(conversation.config)
-				.map(([key, val]) => `${key}: ${val}`)
-				.join(sep) +
-			end;
-
-		const snippets: Snippet[] = [];
-		snippets.push({
-			label: "Install @huggingface/inference",
-			language: "http",
-			code: `npm install --save @huggingface/inference`,
-		});
-		if (conversation.streaming) {
-			snippets.push({
-				label: "Streaming API",
-				needsToken: true,
-				code: `import { HfInference } from "@huggingface/inference"
-
-const client = new HfInference("${tokenStr}")
-
-let out = "";
-
-const stream = client.chatCompletionStream({
-	model: "${conversation.model.id}",
-	messages: ${formattedMessages({ sep: ",\n\t\t", start: "[\n\t\t", end: "\n\t]" })},
-	${formattedConfig({ sep: ",\n\t", start: "", end: "" })}
-});
-
-for await (const chunk of stream) {
-	if (chunk.choices && chunk.choices.length > 0) {
-		const newContent = chunk.choices[0].delta.content;
-		out += newContent;
-		console.log(newContent);
-	}
-}`,
-			});
-		} else {
-			// non-streaming
-			snippets.push({
-				label: "Non-Streaming API",
-				needsToken: true,
-				code: `import { HfInference } from '@huggingface/inference'
-
-const client = new HfInference("${tokenStr}")
-
-const chatCompletion = await client.chatCompletion({
-	model: "${conversation.model.id}",
-	messages: ${formattedMessages({ sep: ",\n\t\t", start: "[\n\t\t", end: "\n\t]" })},
-	${formattedConfig({ sep: ",\n\t", start: "", end: "" })}
-});
-
-console.log(chatCompletion.choices[0].message);`,
-			});
-		}
-
-		return snippets;
-	}
-
-	function getJavascriptSnippetsOAIClient(conversation: Conversation, tokenStr: string) {
-		const formattedMessages = ({ sep, start, end }: MessagesJoiner) =>
-			start +
-			getMessages()
-				.map(({ role, content }) => `{ role: "${role}", content: "${content}" }`)
-				.join(sep) +
-			end;
-
-		const formattedConfig = ({ sep, start, end }: MessagesJoiner) =>
-			start +
-			Object.entries(conversation.config)
-				.map(([key, val]) => `${key}: ${val}`)
-				.join(sep) +
-			end;
-
-		const snippets: Snippet[] = [];
-		snippets.push({
-			label: "Install openai",
-			language: "http",
-			code: `npm install --save openai`,
-		});
-		if (conversation.streaming) {
-			snippets.push({
-				label: "Streaming API",
-				needsToken: true,
-				code: `import { OpenAI } from "openai"
-
-const client = new OpenAI({
-	baseURL: "https://api-inference.huggingface.co/v1/",
-    apiKey: "${tokenStr}"
-})
-
-let out = "";
-
-const stream = await client.chat.completions.create({
-	model: "${conversation.model.id}",
-	messages: ${formattedMessages({ sep: ",\n\t\t", start: "[\n\t\t", end: "\n\t]" })},
-	${formattedConfig({ sep: ",\n\t", start: "", end: "" })},
-	stream: true,
-});
-
-for await (const chunk of stream) {
-	if (chunk.choices && chunk.choices.length > 0) {
-		const newContent = chunk.choices[0].delta.content;
-		out += newContent;
-		console.log(newContent);
-	}
-}`,
-			});
-		} else {
-			// non-streaming
-			snippets.push({
-				label: "Non-Streaming API",
-				needsToken: true,
-				code: `import { OpenAI } from "openai"
-
-const client = new OpenAI({
-    baseURL: "https://api-inference.huggingface.co/v1/",
-    apiKey: "${tokenStr}"
-})
-
-const chatCompletion = await client.chat.completions.create({
-	model: "${conversation.model.id}",
-	messages: ${formattedMessages({ sep: ",\n\t\t", start: "[\n\t\t", end: "\n\t]" })},
-	${formattedConfig({ sep: ",\n\t", start: "", end: "" })}
-});
-
-console.log(chatCompletion.choices[0].message);`,
-			});
-		}
-
-		return snippets;
-	}
-
-	function getPythonSnippetsHfClient(conversation: Conversation, tokenStr: string) {
-		const formattedMessages = ({ sep, start, end }: MessagesJoiner) =>
-			start +
-			getMessages()
-				.map(({ role, content }) => `{ "role": "${role}", "content": "${content}" }`)
-				.join(sep) +
-			end;
-
-		const formattedConfig = ({ sep, start, end, connector }: MessagesJoiner & { connector: string }) =>
-			start +
-			Object.entries(conversation.config)
-				.map(([key, val]) => `${key}${connector}${val}`)
-				.join(sep) +
-			end;
-
-		const snippets: Snippet[] = [];
-		snippets.push({
-			label: "Install the latest huggingface_hub",
-			language: "http",
-			code: `pip install huggingface_hub --upgrade`,
-		});
-		if (conversation.streaming) {
-			snippets.push({
-				label: "Streaming API",
-				needsToken: true,
-				code: `from huggingface_hub import InferenceClient
-
-client = InferenceClient(api_key="${tokenStr}")
-
-messages = ${formattedMessages({ sep: ",\n\t", start: `[\n\t`, end: `\n]` })}
-
-stream = client.chat.completions.create(
-    model="${conversation.model.id}",
-	messages=messages,
-	${formattedConfig({ sep: ",\n\t", start: "", end: "", connector: "=" })},
-	stream=True
-)
-
-for chunk in stream:
-    print(chunk.choices[0].delta.content)`,
-			});
-		} else {
-			// non-streaming
-			snippets.push({
-				label: "Non-Streaming API",
-				needsToken: true,
-				code: `from huggingface_hub import InferenceClient
-
-client = InferenceClient(api_key="${tokenStr}")
-
-messages = ${formattedMessages({ sep: ",\n\t", start: `[\n\t`, end: `\n]` })}
-
-completion = client.chat.completions.create(
-    model="${conversation.model.id}",
-	messages=messages,
-	${formattedConfig({ sep: ",\n\t", start: "", end: "", connector: "=" })}
-)
-
-print(completion.choices[0].message)`,
-			});
-		}
-
-		return snippets;
-	}
-
-	function getPythonSnippetsOAIClient(conversation: Conversation, tokenStr: string) {
-		const formattedMessages = ({ sep, start, end }: MessagesJoiner) =>
-			start +
-			getMessages()
-				.map(({ role, content }) => `{ "role": "${role}", "content": "${content}" }`)
-				.join(sep) +
-			end;
-
-		const formattedConfig = ({ sep, start, end, connector }: MessagesJoiner & { connector: string }) =>
-			start +
-			Object.entries(conversation.config)
-				.map(([key, val]) => `${key}${connector}${val}`)
-				.join(sep) +
-			end;
-
-		const snippets: Snippet[] = [];
-		snippets.push({
-			label: "Install the latest openai",
-			language: "http",
-			code: `pip install openai --upgrade`,
-		});
-		if (conversation.streaming) {
-			snippets.push({
-				label: "Streaming API",
-				needsToken: true,
-				code: `from openai import OpenAI
-
-client = OpenAI(
-	base_url="https://api-inference.huggingface.co/v1/",
-	api_key="${tokenStr}"
-)
-
-messages = ${formattedMessages({ sep: ",\n\t", start: `[\n\t`, end: `\n]` })}
-
-stream = client.chat.completions.create(
-    model="${conversation.model.id}",
-	messages=messages,
-	${formattedConfig({ sep: ",\n\t", start: "", end: "", connector: "=" })},
-	stream=True
-)
-
-for chunk in stream:
-    print(chunk.choices[0].delta.content)`,
-			});
-		} else {
-			// non-streaming
-			snippets.push({
-				label: "Non-Streaming API",
-				needsToken: true,
-				code: `from openai import OpenAI
-
-client = OpenAI(
-	base_url="https://api-inference.huggingface.co/v1/",
-	api_key="${tokenStr}"
-)
-
-messages = ${formattedMessages({ sep: ",\n\t", start: `[\n\t`, end: `\n]` })}
-
-completion = client.chat.completions.create(
-    model="${conversation.model.id}",
-	messages=messages,
-	${formattedConfig({ sep: ",\n\t", start: "", end: "", connector: "=" })}
-)
-
-print(completion.choices[0].message)`,
-			});
-		}
-
-		return snippets;
-	}
-
-	function getHttpSnippets(conversation: Conversation, tokenStr: string) {
-		if (tokenStr === "YOUR_HF_TOKEN") {
-			tokenStr = "{YOUR_HF_TOKEN}";
-		}
-		const formattedMessages = ({ sep, start, end }: MessagesJoiner) =>
-			start +
-			getMessages()
-				.map(({ role, content }) => {
-					// escape single quotes since single quotes is used to define http post body inside curl requests
-					content = content?.replace(/'/g, "'\\''");
-					return `{ "role": "${role}", "content": "${content}" }`;
-				})
-				.join(sep) +
-			end;
-
-		const formattedConfig = ({ sep, start, end }: MessagesJoiner) =>
-			start +
-			Object.entries(conversation.config)
-				.map(([key, val]) => `"${key}": ${val}`)
-				.join(sep) +
-			end;
-
-		const snippets: Snippet[] = [];
-
-		if (conversation.streaming) {
-			snippets.push({
-				label: "Streaming API",
-				needsToken: true,
-				code: `curl 'https://api-inference.huggingface.co/models/${conversation.model.id}/v1/chat/completions' \\
---header "Authorization: Bearer ${tokenStr}" \\
---header 'Content-Type: application/json' \\
---data '{
-    "model": "${conversation.model.id}",
-    "messages": ${formattedMessages({ sep: ",\n    ", start: `[\n    `, end: `\n]` })},
-    ${formattedConfig({ sep: ",\n    ", start: "", end: "" })},
-    "stream": true
-}'`,
-			});
-		} else {
-			// non-streaming
-			snippets.push({
-				label: "Non-Streaming API",
-				needsToken: true,
-				code: `curl 'https://api-inference.huggingface.co/models/${conversation.model.id}/v1/chat/completions' \\
---header "Authorization: Bearer ${tokenStr}" \\
---header 'Content-Type: application/json' \\
---data '{
-    "model": "${conversation.model.id}",
-    "messages": ${formattedMessages({ sep: ",\n    ", start: `[\n    `, end: `\n]` })},
-    ${formattedConfig({ sep: ",\n    ", start: "", end: "" })}
-}'`,
-			});
-		}
-
-		return snippets;
+		return hljs.highlight(code, { language: language === "curl" ? "http" : language }).value;
 	}
 
 	onDestroy(() => {
-		if (timeout) {
-			clearTimeout(timeout);
-		}
+		clearTimeout(timeout);
 	});
 </script>
 
@@ -468,55 +97,52 @@ print(completion.choices[0].message)`,
 		</ul>
 	</div>
 
-	{#if clientSnippetsByLang[selectedLanguage]?.length ?? 0 > 1}
+	{#if (clientSnippetsByLang[selectedLanguage]?.length ?? 0) > 1}
 		<div class="flex gap-x-2 px-2 pt-6">
-			{#each clientSnippetsByLang[selectedLanguage] ?? [] as { name }, idx}
+			{#each clientSnippetsByLang[selectedLanguage] ?? [] as { client }, idx}
+				{@const isActive = idx === selectedClientIdxByLang[selectedLanguage]}
 				<button
-					class="rounded-md px-1.5 py-0.5 leading-tight {idx === selectedClientIdxByLang[selectedLanguage]
-						? 'bg-black text-gray-100 dark:bg-gray-600 dark:text-white'
-						: 'text-gray-500 hover:text-gray-600 dark:hover:text-gray-400'}"
-					on:click={() => (selectedClientIdxByLang[selectedLanguage] = idx)}>{name}</button
+					class="rounded-lg border px-1.5 py-0.5 text-sm leading-tight
+					{isActive
+						? 'bg-black text-gray-100 dark:border-gray-500 dark:bg-gray-700 dark:text-white'
+						: 'text-gray-500 hover:text-gray-600 dark:border-gray-600 dark:hover:text-gray-400'}"
+					on:click={() => (selectedClientIdxByLang[selectedLanguage] = idx)}>{client}</button
 				>
 			{/each}
 		</div>
 	{/if}
 
-	{#each clientSnippetsByLang[selectedLanguage] ?? [] as { snippets }, idx}
+	{#each clientSnippetsByLang[selectedLanguage] ?? [] as { language, content }, idx}
 		{#if idx === selectedClientIdxByLang[selectedLanguage]}
-			{#each snippets as { label, code, language, needsToken }}
-				<div class="flex items-center justify-between px-2 pt-6 pb-4">
-					<h2 class="font-semibold">{label}</h2>
-					<div class="flex items-center gap-x-4">
-						{#if needsToken && $token.value}
-							<label class="flex items-center gap-x-1.5 text-sm select-none">
-								<input type="checkbox" bind:checked={showToken} />
-								<p class="leading-none">With token</p>
-							</label>
-						{/if}
-						<button
-							class="flex items-center gap-x-2 rounded-md border bg-white px-1.5 py-0.5 text-sm shadow-xs transition dark:border-gray-800 dark:bg-gray-800"
-							on:click={e => {
-								const el = e.currentTarget;
-								el.classList.add("text-green-500");
-								navigator.clipboard.writeText(code);
-								if (timeout) {
-									clearTimeout(timeout);
-								}
-								timeout = setTimeout(() => {
-									el.classList.remove("text-green-500");
-								}, 400);
-							}}
-						>
-							<IconCopyCode classNames="text-xs" /> Copy code
-						</button>
-					</div>
+			<div class="flex items-center justify-end px-2 pt-6 pb-4">
+				<div class="flex items-center gap-x-4">
+					<label class="flex items-center gap-x-1.5 text-sm select-none">
+						<input type="checkbox" bind:checked={showToken} />
+						<p class="leading-none">With token</p>
+					</label>
+					<button
+						class="flex items-center gap-x-2 rounded-md border bg-white px-1.5 py-0.5 text-sm shadow-xs transition dark:border-gray-800 dark:bg-gray-800"
+						on:click={e => {
+							const el = e.currentTarget;
+							el.classList.add("text-green-500");
+							navigator.clipboard.writeText(content);
+							if (timeout) {
+								clearTimeout(timeout);
+							}
+							timeout = setTimeout(() => {
+								el.classList.remove("text-green-500");
+							}, 400);
+						}}
+					>
+						<IconCopyCode classNames="text-xs" /> Copy code
+					</button>
 				</div>
-				<pre
-					class="overflow-x-auto rounded-lg border border-gray-200/80 bg-white px-4 py-6 text-sm shadow-xs dark:border-gray-800 dark:bg-gray-800/50">{@html highlight(
-						code,
-						language ?? selectedLanguage
-					)}</pre>
-			{/each}
+			</div>
+			<pre
+				class="overflow-x-auto rounded-lg border border-gray-200/80 bg-white px-4 py-6 text-sm shadow-xs dark:border-gray-800 dark:bg-gray-800/50">{@html highlight(
+					content,
+					language ?? selectedLanguage
+				)}</pre>
 		{/if}
 	{/each}
 </div>

src/lib/components/InferencePlayground/InferencePlaygroundConversation.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import type { Conversation } from "$lib/components/InferencePlayground/types";
+	import type { Conversation } from "$lib/types";
 
 	import { createEventDispatcher, tick } from "svelte";
 

src/lib/components/InferencePlayground/InferencePlaygroundConversationHeader.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import type { Conversation, ModelEntryWithTokenizer } from "$lib/components/InferencePlayground/types";
+	import type { Conversation, ModelWithTokenizer } from "$lib/types";
 
 	import { createEventDispatcher } from "svelte";
 
@@ -17,7 +17,7 @@
 
 	let modelSelectorOpen = false;
 
-	function changeModel(newModelId: ModelEntryWithTokenizer["id"]) {
+	function changeModel(newModelId: ModelWithTokenizer["id"]) {
 		const model = $models.find(m => m.id === newModelId);
 		if (!model) {
 			return;

src/lib/components/InferencePlayground/InferencePlaygroundGenerationConfig.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import type { Conversation } from "$lib/components/InferencePlayground/types";
+	import type { Conversation } from "$lib/types";
 
 	import { GENERATION_CONFIG_KEYS, GENERATION_CONFIG_SETTINGS } from "./generationConfigSettings";
 	import { customMaxTokens } from "./inferencePlaygroundUtils";

src/lib/components/InferencePlayground/InferencePlaygroundMessage.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
 	import { createEventDispatcher } from "svelte";
-	import type { ConversationMessage } from "./types";
+	import type { ConversationMessage } from "$lib/types";
 
 	export let message: ConversationMessage;
 	export let loading: boolean = false;

src/lib/components/InferencePlayground/InferencePlaygroundModelSelector.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import type { Conversation, ModelEntryWithTokenizer } from "./types";
+	import type { Conversation, ModelWithTokenizer } from "$lib/types";
 
 	import { models } from "$lib/stores/models";
 	import Avatar from "../Avatar.svelte";
@@ -13,7 +13,7 @@
 	let showModelPickerModal = false;
 
 	// Model
-	function changeModel(modelId: ModelEntryWithTokenizer["id"]) {
+	function changeModel(modelId: ModelWithTokenizer["id"]) {
 		const model = $models.find(m => m.id === modelId);
 		if (!model) {
 			return;

src/lib/components/InferencePlayground/InferencePlaygroundModelSelectorModal.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import type { Conversation } from "./types";
+	import type { Conversation } from "$lib/types";
 
 	import { createEventDispatcher, tick } from "svelte";
 

src/lib/components/InferencePlayground/InferencePlaygroundProviderSelect.svelte

@@ -1,32 +1,25 @@
 <script lang="ts">
-	import type { Conversation } from "./types";
+	import type { Conversation } from "$lib/types";
 
-	import { browser } from "$app/environment";
-	import { fetchHuggingFaceModel, type InferenceProviderMapping } from "$lib/fetchers/providers";
-	import { token } from "$lib/stores/token";
 	import { randomPick } from "$lib/utils/array";
+	import { cn } from "$lib/utils/cn";
 	import { createSelect, createSync } from "@melt-ui/svelte";
+	import { onMount } from "svelte";
 	import IconCaret from "../Icons/IconCaret.svelte";
 	import IconProvider from "../Icons/IconProvider.svelte";
-	import { cn } from "$lib/utils/cn";
 
 	export let conversation: Conversation;
 	let classes: string | undefined = undefined;
 	export { classes as class };
 
-	async function loadProviders(modelId: string) {
-		if (!browser) return;
-		providerMap = {};
-		const res = await fetchHuggingFaceModel(modelId, $token.value);
-		providerMap = res.inferenceProviderMapping;
-		// Commented out. I'm not sure we want to maintain, or always random pick
-		if ((conversation.provider ?? "") in providerMap) return;
-		conversation.provider = randomPick(Object.keys(providerMap));
+	function reset(providers: typeof conversation.model.inferenceProviderMapping) {
+		const validProvider = providers.find(p => p.provider === conversation.provider);
+		if (validProvider) return;
+		conversation.provider = randomPick(providers)?.provider;
 	}
 
-	let providerMap: InferenceProviderMapping = {};
-	$: modelId = conversation.model.id;
-	$: loadProviders(modelId);
+	$: providers = conversation.model.inferenceProviderMapping;
+	$: reset(providers);
 
 	const {
 		elements: { trigger, menu, option },
@@ -96,7 +89,7 @@
 	</button>
 
 	<div {...$menu} use:menu class="rounded-lg border bg-gray-100/80 dark:border-gray-700 dark:bg-gray-800">
-		{#each Object.keys(providerMap) as provider (provider)}
+		{#each conversation.model.inferenceProviderMapping as { provider }}
 			<div {...$option({ value: provider })} use:option class="group p-1 text-sm dark:text-white">
 				<div
 					class="flex items-center gap-2 rounded-md px-2 py-1.5 group-data-[highlighted]:bg-gray-200 dark:group-data-[highlighted]:bg-gray-700"

src/lib/components/InferencePlayground/inferencePlaygroundUtils.ts

@@ -1,7 +1,9 @@
 import { type ChatCompletionOutputMessage } from "@huggingface/tasks";
-import type { Conversation, ModelEntryWithTokenizer } from "./types";
+import type { InferenceSnippet, ModelDataMinimal } from "@huggingface/tasks";
+import type { Conversation, ModelWithTokenizer } from "$lib/types";
 
-import { HfInference } from "@huggingface/inference";
+import { HfInference, snippets, type InferenceProvider } from "@huggingface/inference";
+import { keys } from "$lib/utils/object";
 
 export async function handleStreamingResponse(
 	hf: HfInference,
@@ -57,7 +59,7 @@ export async function handleNonStreamingResponse(
 	throw new Error("No response from the model");
 }
 
-export function isSystemPromptSupported(model: ModelEntryWithTokenizer) {
+export function isSystemPromptSupported(model: ModelWithTokenizer) {
 	return model?.tokenizerConfig?.chat_template?.includes("system");
 }
 
@@ -125,3 +127,45 @@ export const customMaxTokens: { [key: string]: number } = {
 	"meta-llama/Llama-3.1-70B-Instruct": 32768,
 	"meta-llama/Llama-3.1-8B-Instruct": 8192,
 } as const;
+
+// Order of the elements in InferenceModal.svelte is determined by this const
+export const inferenceSnippetLanguages = ["python", "js", "curl"] as const;
+
+export type InferenceSnippetLanguage = (typeof inferenceSnippetLanguages)[number];
+
+const GET_SNIPPET_FN = {
+	curl: snippets.curl.getCurlInferenceSnippet,
+	js: snippets.js.getJsInferenceSnippet,
+	python: snippets.python.getPythonInferenceSnippet,
+} as const;
+
+export type GetInferenceSnippetReturn = (InferenceSnippet & { language: InferenceSnippetLanguage })[];
+
+export function getInferenceSnippet(
+	model: ModelWithTokenizer,
+	provider: InferenceProvider,
+	language: InferenceSnippetLanguage,
+	accessToken: string,
+	opts?: Record<string, unknown>
+): GetInferenceSnippetReturn {
+	const providerId = model.inferenceProviderMapping.find(p => p.provider === provider)?.providerId;
+	const snippetsByClient = GET_SNIPPET_FN[language](
+		{ ...model, inference: "" },
+		accessToken,
+		provider,
+		providerId,
+		opts
+	);
+	return snippetsByClient.map(snippetByClient => ({ ...snippetByClient, language }));
+}
+
+/**
+ * - If language is defined, the function checks if in an inference snippet is available for that specific language
+ */
+export function hasInferenceSnippet(
+	model: ModelWithTokenizer,
+	provider: InferenceProvider,
+	language: InferenceSnippetLanguage
+): boolean {
+	return getInferenceSnippet(model, provider, language, "").length > 0;
+}

src/lib/components/InferencePlayground/types.ts

@@ -1,27 +0,0 @@
-import type { GenerationConfig } from "$lib/components/InferencePlayground/generationConfigSettings";
-import type { ModelEntry } from "@huggingface/hub";
-import type { ChatCompletionInputMessage } from "@huggingface/tasks";
-
-export type ConversationMessage = Omit<ChatCompletionInputMessage, "content"> & { content?: string };
-
-export type Conversation = {
-	model: ModelEntryWithTokenizer;
-	config: GenerationConfig;
-	messages: ConversationMessage[];
-	systemMessage: ConversationMessage;
-	streaming: boolean;
-	provider?: string;
-};
-
-export type Session = {
-	conversations: [Conversation] | [Conversation, Conversation];
-};
-
-interface TokenizerConfig {
-	chat_template?: string;
-	model_max_length?: number;
-}
-
-export interface ModelEntryWithTokenizer extends ModelEntry {
-	tokenizerConfig: TokenizerConfig;
-}

src/lib/fetchers/providers.ts

@@ -1,77 +0,0 @@
-export interface HuggingFaceModelResponse {
-	_id: string;
-	id: string;
-	inferenceProviderMapping: InferenceProviderMapping;
-}
-
-export type InferenceProviderMapping = {
-	[k: string]: Provider;
-};
-
-export interface Provider {
-	status: string;
-	providerId: string;
-	task: string;
-}
-
-/**
- * Error thrown when the Hugging Face API request fails
- */
-export class HuggingFaceApiError extends Error {
-	status: number;
-	details: string;
-
-	constructor(message: string, status: number, details: string) {
-		super(message);
-		this.name = "HuggingFaceApiError";
-		this.status = status;
-		this.details = details;
-	}
-}
-
-/**
- * Fetches model data from the Hugging Face API
- *
- * @param modelId - The Hugging Face model ID (can include namespace like "username/model-name")
- * @param token - Optional Hugging Face API token for authentication
- * @returns Promise resolving to the model data
- * @throws {HuggingFaceApiError} When the API request fails
- */
-export async function fetchHuggingFaceModel(modelId: string, token?: string): Promise<HuggingFaceModelResponse> {
-	if (!modelId) {
-		throw new Error("Model ID is required");
-	}
-
-	// Construct the API URL
-	const apiUrl = `https://huggingface.co/api/models/${modelId}?expand%5B%5D=inferenceProviderMapping`;
-
-	// Prepare headers for the request
-	const headers: HeadersInit = {};
-	if (token) {
-		headers["Authorization"] = `Bearer ${token}`;
-	}
-
-	try {
-		// Make the request to Hugging Face API
-		const response = await fetch(apiUrl, {
-			method: "GET",
-			headers,
-		});
-
-		if (!response.ok) {
-			const errorText = await response.text();
-			throw new HuggingFaceApiError("Failed to fetch data from Hugging Face API", response.status, errorText);
-		}
-
-		return (await response.json()) as HuggingFaceModelResponse;
-	} catch (error) {
-		if (error instanceof HuggingFaceApiError) {
-			throw error;
-		}
-
-		// Handle other errors (network, etc.)
-		throw new Error(
-			`Error fetching Hugging Face model data: ${error instanceof Error ? error.message : "Unknown error"}`
-		);
-	}
-}

src/lib/stores/models.ts

@@ -1,8 +1,8 @@
 import { page } from "$app/stores";
-import type { ModelEntryWithTokenizer } from "$lib/components/InferencePlayground/types";
+import type { ModelWithTokenizer } from "$lib/types";
 import { readable } from "svelte/store";
 
-export const models = readable<ModelEntryWithTokenizer[]>(undefined, set => {
+export const models = readable<ModelWithTokenizer[]>(undefined, set => {
 	const unsub = page.subscribe($p => set($p.data.models));
 	return unsub;
 });

src/lib/stores/session.ts

@@ -5,7 +5,7 @@ import {
 	defaultSystemMessage,
 	FEATURED_MODELS_IDS,
 } from "$lib/components/InferencePlayground/inferencePlaygroundUtils";
-import type { Conversation, ConversationMessage, Session } from "$lib/components/InferencePlayground/types";
+import { PipelineTag, type Conversation, type ConversationMessage, type Session } from "$lib/types";
 
 import { models } from "$lib/stores/models";
 import { get, writable } from "svelte/store";
@@ -30,14 +30,18 @@ function createSessionStore() {
 				{
 					model: get(models).find(m => FEATURED_MODELS_IDS.includes(m.id)) ??
 						get(models)[0] ?? {
+							_id: "",
+							inferenceProviderMapping: [],
+							pipeline_tag: PipelineTag.TextGeneration,
+							trendingScore: 0,
+							tags: ["text-generation"],
 							id: "",
-							downloads: 0,
-							gated: false,
-							likes: 0,
-							name: "",
-							private: false,
 							tokenizerConfig: {},
-							updatedAt: new Date(),
+							config: {
+								architectures: [] as string[],
+								model_type: "",
+								tokenizer_config: {},
+							},
 						},
 					config: { ...defaultGenerationConfig },
 					messages: [{ ...startMessageUser }],

src/lib/types.ts

@@ -0,0 +1,158 @@
+import type { GenerationConfig } from "$lib/components/InferencePlayground/generationConfigSettings";
+import type { ChatCompletionInputMessage } from "@huggingface/tasks";
+
+export type ConversationMessage = Omit<ChatCompletionInputMessage, "content"> & { content?: string };
+
+export type Conversation = {
+	model: ModelWithTokenizer;
+	config: GenerationConfig;
+	messages: ConversationMessage[];
+	systemMessage: ConversationMessage;
+	streaming: boolean;
+	provider?: string;
+};
+
+export type Session = {
+	conversations: [Conversation] | [Conversation, Conversation];
+};
+
+interface TokenizerConfig {
+	chat_template?: string;
+	model_max_length?: number;
+}
+
+export type ModelWithTokenizer = Model & {
+	tokenizerConfig: TokenizerConfig;
+};
+
+export type Model = {
+	_id: string;
+	id: string;
+	inferenceProviderMapping: InferenceProviderMapping[];
+	trendingScore: number;
+	config: Config;
+	tags: string[];
+	pipeline_tag: PipelineTag;
+	library_name?: LibraryName;
+};
+
+export type Config = {
+	architectures: string[];
+	model_type: string;
+	tokenizer_config: TokenizerConfig;
+	auto_map?: AutoMap;
+	quantization_config?: QuantizationConfig;
+};
+
+export type AutoMap = {
+	AutoConfig: string;
+	AutoModel?: string;
+	AutoModelForCausalLM: string;
+	AutoModelForSequenceClassification?: string;
+	AutoModelForTokenClassification?: string;
+	AutoModelForQuestionAnswering?: string;
+};
+
+export type QuantizationConfig = {
+	quant_method: string;
+	bits?: number;
+};
+
+// export type TokenizerConfig = {
+// 	bos_token?: Token | BosTokenEnum | null;
+// 	chat_template: ChatTemplateElement[] | string;
+// 	eos_token: Token | EOSTokenEnum;
+// 	pad_token?: Token | null | string;
+// 	unk_token?: Token | UnkTokenEnum | null;
+// 	use_default_system_prompt?: boolean;
+// };
+
+export type Token = {
+	__type: Type;
+	content: Content;
+	lstrip: boolean;
+	normalized: boolean;
+	rstrip: boolean;
+	single_word: boolean;
+};
+
+export enum Type {
+	AddedToken = "AddedToken",
+}
+
+export enum Content {
+	BeginOfSentence = "<｜begin▁of▁sentence｜>",
+	ContentS = "</s>",
+	EndOfSentence = "<｜end▁of▁sentence｜>",
+	S = "<s>",
+	Unk = "<unk>",
+}
+
+export enum BosTokenEnum {
+	BeginOfText = "<|begin_of_text|>",
+	Bos = "<bos>",
+	BosToken = "<BOS_TOKEN>",
+	Endoftext = "<|endoftext|>",
+	IMStart = "<|im_start|>",
+	S = "<s>",
+	Startoftext = "<|startoftext|>",
+}
+
+export type ChatTemplateElement = {
+	name: string;
+	template: string;
+};
+
+export enum EOSTokenEnum {
+	EOS = "<eos>",
+	EndOfText = "<|end_of_text|>",
+	EndOfTurnToken = "<|END_OF_TURN_TOKEN|>",
+	Endoftext = "<|endoftext|>",
+	EotID = "<|eot_id|>",
+	IMEnd = "<|im_end|>",
+	S = "</s>",
+}
+
+export enum UnkTokenEnum {
+	Endoftext = "<|endoftext|>",
+	Unk = "<unk>",
+}
+
+export type InferenceProviderMapping = {
+	provider: Provider;
+	providerId: string;
+	status: Status;
+	task: Task;
+};
+
+export enum Provider {
+	Cerebras = "cerebras",
+	FalAI = "fal-ai",
+	FireworksAI = "fireworks-ai",
+	HFInference = "hf-inference",
+	Hyperbolic = "hyperbolic",
+	Nebius = "nebius",
+	Novita = "novita",
+	Replicate = "replicate",
+	Sambanova = "sambanova",
+	Together = "together",
+}
+
+export enum Status {
+	Live = "live",
+	Staging = "staging",
+}
+
+export enum Task {
+	Conversational = "conversational",
+}
+
+export enum LibraryName {
+	Mlx = "mlx",
+	Transformers = "transformers",
+	Vllm = "vllm",
+}
+
+export enum PipelineTag {
+	TextGeneration = "text-generation",
+}

src/lib/utils/object.ts

@@ -0,0 +1,4 @@
+// typed Object.keys
+export function keys<T extends object>(o: T): (keyof T)[] {
+	return Object.keys(o) as (keyof T)[];
+}

src/routes/+page.server.ts

@@ -1,11 +1,10 @@
-import type { ModelEntryWithTokenizer } from "$lib/components/InferencePlayground/types";
-import type { ModelEntry } from "@huggingface/hub";
-import type { PageServerLoad } from "./$types";
 import { env } from "$env/dynamic/private";
+import type { Model, ModelWithTokenizer } from "$lib/types";
+import type { PageServerLoad } from "./$types";
 
 export const load: PageServerLoad = async ({ fetch }) => {
 	const apiUrl =
-		"https://huggingface.co/api/models?pipeline_tag=text-generation&inference_provider=hf-inference&filter=conversational";
+		"https://huggingface.co/api/models?pipeline_tag=text-generation&filter=conversational&inference_provider=all&limit=100&expand[]=inferenceProviderMapping&expand[]=config&expand[]=library_name&expand[]=pipeline_tag&expand[]=tags&expand[]=mask_token&expand[]=trendingScore";
 	const HF_TOKEN = env.HF_TOKEN;
 
 	const res = await fetch(apiUrl, {
@@ -17,7 +16,7 @@ export const load: PageServerLoad = async ({ fetch }) => {
 		console.error(`Error fetching warm models`, res.status, res.statusText);
 		return { models: [] };
 	}
-	const compatibleModels: ModelEntry[] = await res.json();
+	const compatibleModels: Model[] = await res.json();
 	compatibleModels.sort((a, b) => a.id.toLowerCase().localeCompare(b.id.toLowerCase()));
 
 	const promises = compatibleModels.map(async model => {
@@ -32,10 +31,10 @@ export const load: PageServerLoad = async ({ fetch }) => {
 			return null; // Ignore failed requests by returning null
 		}
 		const tokenizerConfig = await res.json();
-		return { ...model, tokenizerConfig } satisfies ModelEntryWithTokenizer;
+		return { ...model, tokenizerConfig } satisfies ModelWithTokenizer;
 	});
 
-	const models: ModelEntryWithTokenizer[] = (await Promise.all(promises)).filter(model => model !== null);
+	const models: ModelWithTokenizer[] = (await Promise.all(promises)).filter(model => model !== null);
 
 	return { models };
 };