abstract token to store

src/lib/components/InferencePlayground/InferencePlayground.svelte

@@ -4,7 +4,6 @@
 	import { page } from "$app/stores";
 	import { defaultGenerationConfig } from "./generationConfigSettings";
 	import {
-		createHfInference,
 		FEATURED_MODELS_IDS,
 		handleNonStreamingResponse,
 		handleStreamingResponse,
@@ -13,8 +12,10 @@
 
 	import { goto } from "$app/navigation";
 	import { models } from "$lib/stores/models";
+	import { token } from "$lib/stores/token";
 	import { isMac } from "$lib/utils/platform";
-	import { onDestroy, onMount } from "svelte";
+	import { HfInference } from "@huggingface/inference";
+	import { onDestroy } from "svelte";
 	import IconCode from "../Icons/IconCode.svelte";
 	import IconCompare from "../Icons/IconCompare.svelte";
 	import IconDelete from "../Icons/IconDelete.svelte";
@@ -61,14 +62,11 @@
 		session = session;
 	}
 
-	let hfToken = "";
 	let viewCode = false;
 	let viewSettings = false;
-	let showTokenModal = false;
 	let loading = false;
 	let abortControllers: AbortController[] = [];
 	let waitForNonStreaming = true;
-	let storeLocallyHfToken = true;
 	let selectCompareModelOpen = false;
 
 	interface GenerationStatistics {
@@ -79,8 +77,6 @@
 		| [GenerationStatistics]
 		| [GenerationStatistics, GenerationStatistics];
 
-	const hfTokenLocalStorageKey = "hf_token";
-
 	$: systemPromptSupported = session.conversations.some(conversation => isSystemPromptSupported(conversation.model));
 	$: compareActive = session.conversations.length === 2;
 
@@ -120,15 +116,9 @@
 		waitForNonStreaming = false;
 	}
 
-	function resetToken() {
-		hfToken = "";
-		localStorage.removeItem(hfTokenLocalStorageKey);
-		showTokenModal = true;
-	}
-
 	async function runInference(conversation: Conversation, conversationIdx: number) {
 		const startTime = performance.now();
-		const hf = createHfInference(hfToken);
+		const hf = new HfInference($token.value);
 
 		if (conversation.streaming) {
 			let addStreamingMessage = true;
@@ -170,8 +160,8 @@
 	}
 
 	async function submit() {
-		if (!hfToken) {
-			showTokenModal = true;
+		if (!$token.value) {
+			$token.showModal = true;
 			return;
 		}
 
@@ -201,9 +191,7 @@
 			}
 			if (error instanceof Error) {
 				if (error.message.includes("token seems invalid")) {
-					hfToken = "";
-					localStorage.removeItem(hfTokenLocalStorageKey);
-					showTokenModal = true;
+					token.reset();
 				}
 				if (error.name !== "AbortError") {
 					alert("error: " + error.message);
@@ -229,12 +217,8 @@
 		const submittedHfToken = (formData.get("hf-token") as string).trim() ?? "";
 		const RE_HF_TOKEN = /\bhf_[a-zA-Z0-9]{34}\b/;
 		if (RE_HF_TOKEN.test(submittedHfToken)) {
-			hfToken = submittedHfToken;
-			if (storeLocallyHfToken) {
-				localStorage.setItem(hfTokenLocalStorageKey, JSON.stringify(hfToken));
-			}
+			token.setValue(submittedHfToken);
 			submit();
-			showTokenModal = false;
 		} else {
 			alert("Please provide a valid HF token.");
 		}
@@ -279,13 +263,6 @@
 		}
 	}
 
-	onMount(() => {
-		const storedHfToken = localStorage.getItem(hfTokenLocalStorageKey);
-		if (storedHfToken !== null) {
-			hfToken = JSON.parse(storedHfToken);
-		}
-	});
-
 	onDestroy(() => {
 		for (const abortController of abortControllers) {
 			abortController.abort();
@@ -293,8 +270,12 @@
 	});
 </script>
 
-{#if showTokenModal}
-	<HFTokenModal bind:storeLocallyHfToken on:close={() => (showTokenModal = false)} on:submit={handleTokenSubmit} />
+{#if $token.showModal}
+	<HFTokenModal
+		bind:storeLocallyHfToken={$token.writeToLocalStorage}
+		on:close={() => ($token.showModal = false)}
+		on:submit={handleTokenSubmit}
+	/>
 {/if}
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
@@ -344,7 +325,6 @@
 						{loading}
 						{conversation}
 						{viewCode}
-						{hfToken}
 						{compareActive}
 						on:addMessage={() => addMessage(conversationIdx)}
 						on:deleteMessage={e => deleteMessage(conversationIdx, e.detail)}
@@ -460,9 +440,9 @@
 				</div>
 
 				<GenerationConfig bind:conversation={session.conversations[0]} />
-				{#if hfToken}
+				{#if $token.value}
 					<button
-						on:click={resetToken}
+						on:click={token.reset}
 						class="mt-auto flex items-center gap-1 self-end text-sm text-gray-500 underline decoration-gray-300 hover:text-gray-800 dark:text-gray-400 dark:decoration-gray-600 dark:hover:text-gray-200"
 						><svg xmlns="http://www.w3.org/2000/svg" class="text-xs" width="1em" height="1em" viewBox="0 0 32 32"
 							><path

src/lib/components/InferencePlayground/InferencePlaygroundCodeSnippets.svelte

@@ -9,13 +9,13 @@
 
 	import IconCopyCode from "../Icons/IconCopyCode.svelte";
 	import { isSystemPromptSupported } from "./inferencePlaygroundUtils";
+	import { token } from "$lib/stores/token";
 
 	hljs.registerLanguage("javascript", javascript);
 	hljs.registerLanguage("python", python);
 	hljs.registerLanguage("http", http);
 
 	export let conversation: Conversation;
-	export let hfToken: string;
 
 	const dispatch = createEventDispatcher<{ closeCode: void }>();
 
@@ -66,8 +66,8 @@
 	const selectedClientIdxByLang: Record<Language, number> = Object.fromEntries(lanuages.map(lang => [lang, 0]));
 
 	function getTokenStr(showToken: boolean) {
-		if (hfToken && showToken) {
-			return hfToken;
+		if ($token.value && showToken) {
+			return $token.value;
 		}
 		return "YOUR_HF_TOKEN";
 	}
@@ -487,7 +487,7 @@ print(completion.choices[0].message)`,
 				<div class="flex items-center justify-between px-2 pt-6 pb-4">
 					<h2 class="font-semibold">{label}</h2>
 					<div class="flex items-center gap-x-4">
-						{#if needsToken && hfToken}
+						{#if needsToken && $token.value}
 							<label class="flex items-center gap-x-1.5 text-sm select-none">
 								<input type="checkbox" bind:checked={showToken} />
 								<p class="leading-none">With token</p>

src/lib/components/InferencePlayground/InferencePlaygroundConversation.svelte

@@ -10,7 +10,6 @@
 	export let conversation: Conversation;
 	export let loading: boolean;
 	export let viewCode: boolean;
-	export let hfToken: string;
 	export let compareActive: boolean;
 
 	let shouldScrollToBottom = true;
@@ -101,6 +100,6 @@
 			</div>
 		</button>
 	{:else}
-		<CodeSnippets {conversation} {hfToken} on:closeCode />
+		<CodeSnippets {conversation} on:closeCode />
 	{/if}
 </div>

src/lib/components/InferencePlayground/InferencePlaygroundModelSelector.svelte

@@ -9,6 +9,7 @@
 	import Avatar from "../Avatar.svelte";
 	import { defaultSystemMessage } from "./InferencePlaygroundGenerationConfig.svelte";
 	import { models } from "$lib/stores/models";
+	import { fetchHuggingFaceModel, type Provider } from "$lib/fetchers/providers";
 
 	export let conversation: Conversation;
 
@@ -32,6 +33,13 @@
 	}
 
 	$: [nameSpace, modelName] = conversation.model.id.split("/");
+
+	async function loadProviders(modelId: string) {
+		const providers = await fetchHuggingFaceModel;
+	}
+	let providers: Provider[] = [];
+
+	const id = crypto.randomUUID();
 </script>
 
 {#if showModelPickerModal}
@@ -43,11 +51,33 @@
 {/if}
 
 <div class="flex flex-col gap-2">
-	<label for="countries" class="flex items-baseline text-sm font-medium text-gray-900 dark:text-white"
-		>Models<span class="ml-4 font-normal text-gray-400">{models.length}</span>
+	<label for={id} class="flex items-baseline gap-2 text-sm font-medium text-gray-900 dark:text-white">
+		Models<span class="text-xs font-normal text-gray-400">{$models.length}</span>
+	</label>
+
+	<button
+		{id}
+		class="relative flex items-center justify-between gap-6 overflow-hidden rounded-lg border bg-gray-100/80 px-3 py-1.5 leading-tight whitespace-nowrap shadow-sm hover:brightness-95 dark:border-gray-700 dark:bg-gray-800 dark:hover:brightness-110"
+		on:click={() => (showModelPickerModal = true)}
+	>
+		<div class="flex flex-col items-start">
+			<div class="flex items-center gap-1 text-sm text-gray-500 dark:text-gray-300">
+				<Avatar orgName={nameSpace} size="sm" />
+				{nameSpace}
+			</div>
+			<div>{modelName}</div>
+		</div>
+		<IconCaret classNames="text-xl bg-gray-100 dark:bg-gray-600 rounded-sm size-4 flex-none absolute right-2" />
+	</button>
+</div>
+
+<div class="flex flex-col gap-2">
+	<label for={id} class="flex items-baseline gap-2 text-sm font-medium text-gray-900 dark:text-white">
+		Providers<span class="text-xs font-normal text-gray-400"></span>
 	</label>
 
 	<button
+		{id}
 		class="relative flex items-center justify-between gap-6 overflow-hidden rounded-lg border bg-gray-100/80 px-3 py-1.5 leading-tight whitespace-nowrap shadow-sm hover:brightness-95 dark:border-gray-700 dark:bg-gray-800 dark:hover:brightness-110"
 		on:click={() => (showModelPickerModal = true)}
 	>

src/lib/components/InferencePlayground/InferencePlaygroundProviderSelectorModal.svelte

@@ -0,0 +1,179 @@
+<script lang="ts">
+	import type { Conversation, ModelEntryWithTokenizer } from "./types";
+
+	import { createEventDispatcher, tick } from "svelte";
+
+	import { FEATURED_MODELS_IDS } from "./inferencePlaygroundUtils";
+	import IconSearch from "../Icons/IconSearch.svelte";
+	import IconStar from "../Icons/IconStar.svelte";
+	import { models } from "$lib/stores/models";
+
+	export let conversation: Conversation;
+
+	let backdropEl: HTMLDivElement;
+	let highlightIdx = 0;
+	let ignoreCursorHighlight = false;
+	let containerEl: HTMLDivElement;
+
+	const dispatch = createEventDispatcher<{ modelSelected: string; close: void }>();
+
+	let featuredModels = $models.filter(m => FEATURED_MODELS_IDS.includes(m.id));
+	let otherModels = $models.filter(m => !FEATURED_MODELS_IDS.includes(m.id));
+
+	if (featuredModels.findIndex(model => model.id === conversation.model.id) !== -1) {
+		highlightIdx = featuredModels.findIndex(model => model.id === conversation.model.id);
+	} else {
+		highlightIdx = featuredModels.length + otherModels.findIndex(model => model.id === conversation.model.id);
+	}
+
+	function handleKeydown(event: KeyboardEvent) {
+		const { key } = event;
+		let scrollLogicalPosition: ScrollLogicalPosition = "end";
+		if (key === "Escape") {
+			event.preventDefault();
+			dispatch("close");
+		} else if (key === "Enter") {
+			event.preventDefault();
+			const highlightedEl = document.querySelector(".highlighted");
+			if (highlightedEl) {
+				(highlightedEl as HTMLButtonElement).click();
+			}
+		} else if (key === "ArrowUp") {
+			event.preventDefault();
+			highlightIdx--;
+			scrollLogicalPosition = "start";
+			ignoreCursorHighlight = true;
+		} else if (key === "ArrowDown") {
+			event.preventDefault();
+			highlightIdx++;
+			ignoreCursorHighlight = true;
+		}
+		const n = featuredModels.length + otherModels.length;
+		highlightIdx = ((highlightIdx % n) + n) % n;
+		scrollToResult(scrollLogicalPosition);
+	}
+
+	async function scrollToResult(block: ScrollLogicalPosition) {
+		await tick();
+		const highlightedEl = document.querySelector(".highlighted");
+		if (containerEl && highlightedEl) {
+			const { bottom: containerBottom, top: containerTop } = containerEl.getBoundingClientRect();
+			const { bottom: highlightedBottom, top: highlightedTop } = highlightedEl.getBoundingClientRect();
+			if (highlightedBottom > containerBottom || containerTop > highlightedTop) {
+				highlightedEl.scrollIntoView({ block });
+			}
+		}
+	}
+
+	function highlightRow(idx: number) {
+		if (!ignoreCursorHighlight) {
+			highlightIdx = idx;
+		}
+	}
+
+	function handleBackdropClick(event: MouseEvent) {
+		if (window?.getSelection()?.toString()) {
+			return;
+		}
+		if (event.target === backdropEl) {
+			dispatch("close");
+		}
+	}
+
+	function filterModels(query: string) {
+		featuredModels = $models.filter(m =>
+			query
+				? FEATURED_MODELS_IDS.includes(m.id) && m.id.toLocaleLowerCase().includes(query.toLocaleLowerCase().trim())
+				: FEATURED_MODELS_IDS.includes(m.id)
+		);
+
+		otherModels = $models.filter(m =>
+			query
+				? !FEATURED_MODELS_IDS.includes(m.id) && m.id.toLocaleLowerCase().includes(query.toLocaleLowerCase().trim())
+				: !FEATURED_MODELS_IDS.includes(m.id)
+		);
+	}
+</script>
+
+<svelte:window on:keydown={handleKeydown} on:mousemove={() => (ignoreCursorHighlight = false)} />
+
+<!-- svelte-ignore a11y-no-static-element-interactions a11y-click-events-have-key-events -->
+<div
+	class="fixed inset-0 z-10 flex h-screen items-start justify-center bg-black/85 pt-32"
+	bind:this={backdropEl}
+	on:click|stopPropagation={handleBackdropClick}
+>
+	<div class="flex w-full max-w-[600px] items-start justify-center overflow-hidden p-10 text-left whitespace-nowrap">
+		<div
+			class="flex h-full w-full flex-col overflow-hidden rounded-lg border bg-white text-gray-900 shadow-md dark:border-gray-800 dark:bg-gray-900 dark:text-gray-300"
+			bind:this={containerEl}
+		>
+			<div class="flex items-center border-b px-3 dark:border-gray-800">
+				<IconSearch classNames="mr-2 text-sm" />
+				<!-- svelte-ignore a11y-autofocus -->
+				<input
+					autofocus
+					class="flex h-10 w-full rounded-md bg-transparent py-3 text-sm placeholder-gray-400 outline-hidden"
+					placeholder="Search models ..."
+					on:input={e => filterModels(e.currentTarget.value)}
+				/>
+			</div>
+			<div class="max-h-[300px] overflow-x-hidden overflow-y-auto">
+				{#if featuredModels.length}
+					<div>
+						<div class="px-2 py-1.5 text-xs font-medium text-gray-500">Trending</div>
+						<div>
+							{#each featuredModels as model, idx}
+								{@const [nameSpace, modelName] = model.id.split("/")}
+								<button
+									class="flex w-full cursor-pointer items-center px-2 py-1.5 text-sm {highlightIdx === idx
+										? 'highlighted bg-gray-100 dark:bg-gray-800'
+										: ''}"
+									on:mouseenter={() => highlightRow(idx)}
+									on:click={() => {
+										dispatch("modelSelected", model.id);
+										dispatch("close");
+									}}
+								>
+									<IconStar classNames="lucide lucide-star mr-1.5 size-4 text-yellow-400" />
+									<span class="inline-flex items-center"
+										><span class="text-gray-500 dark:text-gray-400">{nameSpace}</span><span
+											class="mx-1 text-gray-300 dark:text-gray-700">/</span
+										><span class="text-black dark:text-white">{modelName}</span></span
+									>
+								</button>
+							{/each}
+						</div>
+					</div>
+				{/if}
+				{#if otherModels.length}
+					<div>
+						<div class="px-2 py-1.5 text-xs font-medium text-gray-500">Other Models</div>
+						<div>
+							{#each otherModels as model, _idx}
+								{@const [nameSpace, modelName] = model.id.split("/")}
+								{@const idx = featuredModels.length + _idx}
+								<button
+									class="flex w-full cursor-pointer items-center px-2 py-1.5 text-sm {highlightIdx === idx
+										? 'highlighted bg-gray-100 dark:bg-gray-800'
+										: ''}"
+									on:mouseenter={() => highlightRow(idx)}
+									on:click={() => {
+										dispatch("modelSelected", model.id);
+										dispatch("close");
+									}}
+								>
+									<span class="inline-flex items-center"
+										><span class="text-gray-500 dark:text-gray-400">{nameSpace}</span><span
+											class="mx-1 text-gray-300 dark:text-gray-700">/</span
+										><span class="text-black dark:text-white">{modelName}</span></span
+									>
+								</button>
+							{/each}
+						</div>
+					</div>
+				{/if}
+			</div>
+		</div>
+	</div>
+</div>

src/lib/components/InferencePlayground/inferencePlaygroundUtils.ts

@@ -3,10 +3,6 @@ import type { Conversation, ModelEntryWithTokenizer } from "./types";
 
 import { HfInference } from "@huggingface/inference";
 
-export function createHfInference(token: string): HfInference {
-	return new HfInference(token);
-}
-
 export async function handleStreamingResponse(
 	hf: HfInference,
 	conversation: Conversation,

src/lib/fetchers/providers.ts

@@ -0,0 +1,76 @@
+export interface HuggingFaceModelResponse {
+	_id: string;
+	id: string;
+	inferenceProviderMapping: InferenceProviderMapping;
+}
+
+export type InferenceProviderMapping = {
+	[k: string]: Provider;
+};
+
+export interface Provider {
+	status: string;
+	providerId: string;
+	task: string;
+}
+/**
+ * Error thrown when the Hugging Face API request fails
+ */
+export class HuggingFaceApiError extends Error {
+	status: number;
+	details: string;
+
+	constructor(message: string, status: number, details: string) {
+		super(message);
+		this.name = "HuggingFaceApiError";
+		this.status = status;
+		this.details = details;
+	}
+}
+
+/**
+ * Fetches model data from the Hugging Face API
+ *
+ * @param modelId - The Hugging Face model ID (can include namespace like "username/model-name")
+ * @param token - Optional Hugging Face API token for authentication
+ * @returns Promise resolving to the model data
+ * @throws {HuggingFaceApiError} When the API request fails
+ */
+export async function fetchHuggingFaceModel(modelId: string, token?: string): Promise<HuggingFaceModelResponse> {
+	if (!modelId) {
+		throw new Error("Model ID is required");
+	}
+
+	// Construct the API URL
+	const apiUrl = `https://huggingface.co/api/models/${modelId}?expand%5B%5D=inferenceProviderMapping`;
+
+	// Prepare headers for the request
+	const headers: HeadersInit = {};
+	if (token) {
+		headers["Authorization"] = `Bearer ${token}`;
+	}
+
+	try {
+		// Make the request to Hugging Face API
+		const response = await fetch(apiUrl, {
+			method: "GET",
+			headers,
+		});
+
+		if (!response.ok) {
+			const errorText = await response.text();
+			throw new HuggingFaceApiError("Failed to fetch data from Hugging Face API", response.status, errorText);
+		}
+
+		return (await response.json()) as HuggingFaceModelResponse;
+	} catch (error) {
+		if (error instanceof HuggingFaceApiError) {
+			throw error;
+		}
+
+		// Handle other errors (network, etc.)
+		throw new Error(
+			`Error fetching Hugging Face model data: ${error instanceof Error ? error.message : "Unknown error"}`
+		);
+	}
+}

src/lib/stores/token.ts

@@ -0,0 +1,34 @@
+import { browser } from "$app/environment";
+import { writable } from "svelte/store";
+
+const key = "hf_token";
+
+function createTokenStore() {
+	const store = writable({ value: "", writeToLocalStorage: true, showModal: false });
+
+	function setValue(token: string) {
+		store.update(s => {
+			if (s.writeToLocalStorage) localStorage.setItem(key, JSON.stringify(token));
+			return { ...s, value: token, showModal: !token.length };
+		});
+	}
+
+	if (browser) {
+		const storedHfToken = localStorage.getItem(key);
+		if (storedHfToken !== null) {
+			setValue(JSON.parse(storedHfToken));
+		}
+	}
+
+	return {
+		...store,
+		setValue,
+		reset() {
+			setValue("");
+			localStorage.removeItem(key);
+			store.update(s => ({ ...s, showModal: true }));
+		},
+	};
+}
+
+export const token = createTokenStore();

src/lib/utils/effect.ts

@@ -0,0 +1,51 @@
+import type { Stores, StoresValues } from "svelte/store";
+import { derived } from "svelte/store";
+import { safeOnDestroy } from "./lifecycle";
+import { noop } from "./noop";
+
+type EffectOptions = {
+	/**
+	 * Whether to skip the first run
+	 * @default undefined
+	 */
+	skipFirstRun?: boolean;
+};
+
+/**
+ * A utility function that creates an effect from a set of stores and a function.
+ * The effect is automatically cleaned up when the component is destroyed.
+ *
+ * @template S - The type of the stores object
+ * @param stores - The stores object to derive from
+ * @param fn - The function to run when the stores change
+ * @param opts {@link EffectOptions}
+ * @returns A function that can be used to unsubscribe the effect
+ */
+export function effect<S extends Stores>(
+	stores: S,
+	fn: (values: StoresValues<S>) => (() => void) | void,
+	opts: EffectOptions = {}
+): () => void {
+	const { skipFirstRun } = opts;
+	let isFirstRun = true;
+	let cb: (() => void) | void = undefined;
+
+	// Create a derived store that contains the stores object and an onUnsubscribe function
+	const destroy = derived(stores, stores => {
+		cb?.();
+		if (isFirstRun && skipFirstRun) {
+			isFirstRun = false;
+		} else {
+			cb = fn(stores);
+		}
+	}).subscribe(noop);
+
+	const unsub = () => {
+		destroy();
+		cb?.();
+	};
+
+	// Automatically unsubscribe the effect when the component is destroyed
+	safeOnDestroy(unsub);
+	return unsub;
+}

src/lib/utils/lifecycle.ts

@@ -0,0 +1,17 @@
+import { onDestroy, onMount } from "svelte";
+
+export const safeOnMount = (fn: (...args: unknown[]) => unknown) => {
+	try {
+		onMount(fn);
+	} catch {
+		return fn;
+	}
+};
+
+export const safeOnDestroy = (fn: (...args: unknown[]) => unknown) => {
+	try {
+		onDestroy(fn);
+	} catch {
+		return fn;
+	}
+};

src/lib/utils/noop.ts

@@ -0,0 +1,6 @@
+/**
+ * A no operation function (does nothing)
+ */
+export function noop() {
+	// do nothing
+}

src/routes/+page.server.ts

@@ -18,6 +18,7 @@ export const load: PageServerLoad = async ({ fetch }) => {
 		return { models: [] };
 	}
 	const compatibleModels: ModelEntry[] = await res.json();
+	console.log(compatibleModels);
 	compatibleModels.sort((a, b) => a.id.toLowerCase().localeCompare(b.id.toLowerCase()));
 
 	const promises = compatibleModels.map(async model => {