diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..408bae829ffa971ac9a9a2d2722bcf771286b586 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.next/ +node_modules/ +.env +next-env.d.ts +Dockerfile +.dockerignore +Makefile +.github +.git diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000000000000000000000000000000000000..bffb357a7122523ec94045523758c4b825b448ef --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "next/core-web-vitals" +} diff --git a/.example.env b/.example.env new file mode 100644 index 0000000000000000000000000000000000000000..5824abe9fa6ecd81fb34de7bb8b7980dbd94bc4a --- /dev/null +++ b/.example.env @@ -0,0 +1 @@ +VLLM_URL="http://localhost:8000" diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..7654ef158a469df51f653b7bf3dc623a2a67aff4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +ollama-nextjs-ui.gif filter=lfs diff=lfs merge=lfs -text diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..dda2cdc66d387342674c5ef36f57afe164bba247 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,50 @@ +ARG NODE_VERSION=lts +# Install dependencies only when needed +FROM node:$NODE_VERSION-alpine AS builder +ENV YARN_CACHE_FOLDER=/opt/yarncache +WORKDIR /opt/app +COPY package.json yarn.lock ./ +RUN yarn install --frozen-lockfile +# patch logging for requestHandler +RUN sed -Ei \ + -e '/await requestHandler/iconst __start = new Date;' \ + -e '/await requestHandler/aconsole.log(`[${__start.toISOString()}] ${((new Date - __start) / 1000).toFixed(3)} ${req.method} ${req.url}`);' \ + node_modules/next/dist/server/lib/start-server.js + +ENV NODE_ENV=production +ENV NEXT_TELEMETRY_DISABLED=1 +ENV VLLM_URL="http://localhost:8000" +ENV VLLM_API_KEY="" + +COPY . . +RUN yarn build + +# Production image, copy all the files and run next +FROM node:$NODE_VERSION-alpine AS runner +WORKDIR /opt/app + +ENV NODE_ENV production +ENV NEXT_TELEMETRY_DISABLED 1 + +RUN addgroup --system --gid 1001 nodejs +RUN adduser --system --uid 1001 nextjs + +COPY --from=builder /opt/app/public ./public + +# Set the correct permission for prerender cache +RUN mkdir .next +RUN chown nextjs:nodejs .next + +# Automatically leverage output traces to reduce image size +# https://nextjs.org/docs/advanced-features/output-file-tracing +COPY --from=builder --chown=nextjs:nodejs /opt/app/.next/standalone ./ +COPY --from=builder --chown=nextjs:nodejs /opt/app/.next/static ./.next/static + +USER nextjs + +EXPOSE 3000 + +ENV PORT 3000 +# set hostname to localhost +ENV HOSTNAME "0.0.0.0" +CMD ["node", "server.js"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..48534f2daf7a9d04f24ff9d60f4223cd6aa66f28 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Jakob Hoeg Mørk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..89c5dd2a3a81130a0ba5a49fdc660025f598b193 --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +build: + docker buildx build --platform linux/amd64,linux/arm64 . -t yoziru/nextjs-vllm-ui diff --git a/components.json b/components.json new file mode 100644 index 0000000000000000000000000000000000000000..965b5bef118df8ab63c8bde9f46eba2b965fd799 --- /dev/null +++ b/components.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "new-york", + "rsc": true, + "tsx": true, + "tailwind": { + "config": "tailwind.config.ts", + "css": "src/app/globals.css", + "baseColor": "zinc", + "cssVariables": true, + "prefix": "" + }, + "aliases": { + "components": "@/components", + "utils": "@/lib/utils" + } +} \ No newline at end of file diff --git a/next.config.mjs b/next.config.mjs new file mode 100644 index 0000000000000000000000000000000000000000..5e8be7282cefb3df533128df9fc40aa0bb0b766d --- /dev/null +++ b/next.config.mjs @@ -0,0 +1,8 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + // set basepath based on environment + basePath: process.env.NEXT_PUBLIC_BASE_PATH ?? "", + output: "standalone", +}; + +export default nextConfig; diff --git a/ollama-nextjs-ui.gif b/ollama-nextjs-ui.gif new file mode 100644 index 0000000000000000000000000000000000000000..a171f264d5b505077477f4771afa5c5b43a741dc --- /dev/null +++ b/ollama-nextjs-ui.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91d7ea9d057e42d0ce46a3b0b1f550511d3b15581ce6d43bddf3728c28d70658 +size 5797455 diff --git a/package.json b/package.json new file mode 100644 index 0000000000000000000000000000000000000000..270a8aeabd95e1f720f6ad3321304efc8dd8cd7d --- /dev/null +++ b/package.json @@ -0,0 +1,49 @@ +{ + "name": "nextjs-vllm-ui", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "lint": "next lint" + }, + "dependencies": { + "@hookform/resolvers": "^3.3.4", + "@radix-ui/react-dialog": "^1.0.5", + "@radix-ui/react-icons": "^1.3.0", + "@radix-ui/react-slot": "^1.0.2", + "@radix-ui/react-tabs": "^1.0.4", + "@radix-ui/react-tooltip": "^1.0.7", + "ai": "^3.0.15", + "class-variance-authority": "^0.7.0", + "clsx": "^2.1.0", + "mistral-tokenizer-js": "^1.0.0", + "next": "14.1.4", + "next-themes": "^0.3.0", + "openai": "^4.30.0", + "react": "^18", + "react-code-blocks": "^0.1.6", + "react-dom": "^18", + "react-hook-form": "^7.51.2", + "react-textarea-autosize": "^8.5.3", + "sonner": "^1.4.41", + "tailwind-merge": "^2.2.2", + "use-debounce": "^10.0.0", + "use-local-storage-state": "^19.2.0", + "uuid": "^9.0.1" + }, + "devDependencies": { + "@types/node": "^20.11.30", + "@types/react": "^18.2.73", + "@types/react-dom": "^18.2.23", + "@types/uuid": "^9.0.8", + "autoprefixer": "^10.4.19", + "eslint": "^8.57.0", + "eslint-config-next": "14.1.4", + "postcss": "^8.4.38", + "tailwindcss": "^3.4.3", + "typescript": "^5.4.3", + "typescript-eslint": "^7.4.0" + } +} diff --git a/postcss.config.js b/postcss.config.js new file mode 100644 index 0000000000000000000000000000000000000000..12a703d900da8159c30e75acbd2c4d87ae177f62 --- /dev/null +++ b/postcss.config.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +}; diff --git a/public/next.svg b/public/next.svg new file mode 100644 index 0000000000000000000000000000000000000000..5174b28c565c285e3e312ec5178be64fbeca8398 --- /dev/null +++ b/public/next.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/ollama.png b/public/ollama.png new file mode 100644 index 0000000000000000000000000000000000000000..8cd2cf1ed8043caf62e8b069330889c0cf0f5a3b Binary files /dev/null and b/public/ollama.png differ diff --git a/public/vercel.svg b/public/vercel.svg new file mode 100644 index 0000000000000000000000000000000000000000..d2f84222734f27b623d1c80dda3561b04d1284af --- /dev/null +++ b/public/vercel.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..2044571ab2a45070bb21f1e14139a71905f727b5 --- /dev/null +++ b/src/app/api/chat/route.ts @@ -0,0 +1,222 @@ +import { + createParser, + ParsedEvent, + ReconnectInterval, +} from "eventsource-parser"; +import { NextRequest, NextResponse } from "next/server"; +import { + ChatCompletionAssistantMessageParam, + ChatCompletionCreateParamsStreaming, + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, + ChatCompletionUserMessageParam, +} from "openai/resources/index.mjs"; + +import { encodeChat, tokenLimit } from "@/lib/token-counter"; + +const addSystemMessage = ( + messages: ChatCompletionMessageParam[], + systemPrompt?: string +) => { + // early exit if system prompt is empty + if (!systemPrompt || systemPrompt === "") { + return messages; + } + + // add system prompt to the chat (if it's not already there) + // check first message in the chat + if (!messages) { + // if there are no messages, add the system prompt as the first message + messages = [ + { + content: systemPrompt, + role: "system", + }, + ]; + } else if (messages.length === 0) { + // if there are no messages, add the system prompt as the first message + messages.push({ + content: systemPrompt, + role: "system", + }); + } else { + // if there are messages, check if the first message is a system prompt + if (messages[0].role === "system") { + // if the first message is a system prompt, update it + messages[0].content = systemPrompt; + } else { + // if the first message is not a system prompt, add the system prompt as the first message + messages.unshift({ + content: systemPrompt, + role: "system", + }); + } + } + return messages; +}; + +const formatMessages = ( + messages: ChatCompletionMessageParam[] +): ChatCompletionMessageParam[] => { + let mappedMessages: ChatCompletionMessageParam[] = []; + let messagesTokenCounts: number[] = []; + const responseTokens = 512; + const tokenLimitRemaining = tokenLimit - responseTokens; + let tokenCount = 0; + + messages.forEach((m) => { + if (m.role === "system") { + mappedMessages.push({ + role: "system", + content: m.content, + } as ChatCompletionSystemMessageParam); + } else if (m.role === "user") { + mappedMessages.push({ + role: "user", + content: m.content, + } as ChatCompletionUserMessageParam); + } else if (m.role === "assistant") { + mappedMessages.push({ + role: "assistant", + content: m.content, + } as ChatCompletionAssistantMessageParam); + } else { + return; + } + + // ignore typing + // tslint:disable-next-line + const messageTokens = encodeChat([m]); + messagesTokenCounts.push(messageTokens); + tokenCount += messageTokens; + }); + + if (tokenCount <= tokenLimitRemaining) { + return mappedMessages; + } + + // remove the middle messages until the token count is below the limit + while (tokenCount > tokenLimitRemaining) { + const middleMessageIndex = Math.floor(messages.length / 2); + const middleMessageTokens = messagesTokenCounts[middleMessageIndex]; + mappedMessages.splice(middleMessageIndex, 1); + messagesTokenCounts.splice(middleMessageIndex, 1); + tokenCount -= middleMessageTokens; + } + return mappedMessages; +}; + +export async function POST(req: NextRequest): Promise { + try { + const { messages, chatOptions } = await req.json(); + if (!chatOptions.selectedModel || chatOptions.selectedModel === "") { + throw new Error("Selected model is required"); + } + + const baseUrl = process.env.VLLM_URL; + if (!baseUrl) { + throw new Error("VLLM_URL is not set"); + } + const apiKey = process.env.VLLM_API_KEY; + + const formattedMessages = formatMessages( + addSystemMessage(messages, chatOptions.systemPrompt) + ); + + const stream = await getOpenAIStream( + baseUrl, + chatOptions.selectedModel, + formattedMessages, + chatOptions.temperature, + apiKey, + ); + return new NextResponse(stream, { + headers: { "Content-Type": "text/event-stream" }, + }); + } catch (error) { + console.error(error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : "Unknown error", + }, + { status: 500 } + ); + } +} + +const getOpenAIStream = async ( + apiUrl: string, + model: string, + messages: ChatCompletionMessageParam[], + temperature?: number, + apiKey?: string +): Promise> => { + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + const headers = new Headers(); + headers.set("Content-Type", "application/json"); + if (apiKey !== undefined) { + headers.set("Authorization", `Bearer ${apiKey}`); + headers.set("api-key", apiKey); + } + const chatOptions: ChatCompletionCreateParamsStreaming = { + model: model, + // frequency_penalty: 0, + // max_tokens: 2000, + messages: messages, + // presence_penalty: 0, + stream: true, + temperature: temperature ?? 0.5, + // response_format: { + // type: "json_object", + // } + // top_p: 0.95, + }; + const res = await fetch(apiUrl + "/v1/chat/completions", { + headers: headers, + method: "POST", + body: JSON.stringify(chatOptions), + }); + + if (res.status !== 200) { + const statusText = res.statusText; + const responseBody = await res.text(); + console.error(`vLLM API response error: ${responseBody}`); + throw new Error( + `The vLLM API has encountered an error with a status code of ${res.status} ${statusText}: ${responseBody}` + ); + } + + return new ReadableStream({ + async start(controller) { + const onParse = (event: ParsedEvent | ReconnectInterval) => { + if (event.type === "event") { + const data = event.data; + + if (data === "[DONE]") { + controller.close(); + return; + } + + try { + const json = JSON.parse(data); + const text = json.choices[0].delta.content; + const queue = encoder.encode(text); + controller.enqueue(queue); + } catch (e) { + controller.error(e); + } + } + }; + + const parser = createParser(onParse); + + for await (const chunk of res.body as any) { + // An extra newline is required to make AzureOpenAI work. + const str = decoder.decode(chunk).replace("[DONE]\n", "[DONE]\n\n"); + parser.feed(str); + } + }, + }); +}; diff --git a/src/app/api/health/route.ts b/src/app/api/health/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..3c17550c8aff89085dbad22c3f0097cf41f575bf --- /dev/null +++ b/src/app/api/health/route.ts @@ -0,0 +1,5 @@ +import { NextRequest, NextResponse } from "next/server"; + +export async function GET(req: NextRequest): Promise { + return new NextResponse("OK", { status: 200 }); +} diff --git a/src/app/api/models/route.ts b/src/app/api/models/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..75b6d2bcc3ef37694962a9b662e0bdfc02746bab --- /dev/null +++ b/src/app/api/models/route.ts @@ -0,0 +1,55 @@ +import { NextRequest, NextResponse } from "next/server"; + +export async function GET(req: NextRequest): Promise { + const baseUrl = process.env.VLLM_URL; + const apiKey = process.env.VLLM_API_KEY; + const headers = new Headers(); + if (apiKey !== undefined) { + headers.set("Authorization", `Bearer ${apiKey}`); + headers.set("api-key", apiKey); + } + if (!baseUrl) { + throw new Error("VLLM_URL is not set"); + } + + const envModel = process.env.VLLM_MODEL; + if (envModel) { + return NextResponse.json({ + object: "list", + data: [ + { + id: envModel, + }, + ], + }); + } + + try { + const res = await fetch(`${baseUrl}/v1/models`, { + headers: headers, + cache: "no-store", + }); + if (res.status !== 200) { + const statusText = res.statusText; + const responseBody = await res.text(); + console.error(`vLLM /api/models response error: ${responseBody}`); + return NextResponse.json( + { + success: false, + error: statusText, + }, + { status: res.status } + ); + } + return new NextResponse(res.body, res); + } catch (error) { + console.error(error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : "Unknown error", + }, + { status: 500 } + ); + } +} diff --git a/src/app/chats/[id]/page.tsx b/src/app/chats/[id]/page.tsx new file mode 100644 index 0000000000000000000000000000000000000000..a0a012467fe227249951d8d537dbc97fe2df67f1 --- /dev/null +++ b/src/app/chats/[id]/page.tsx @@ -0,0 +1,15 @@ +"use client"; + +import React from "react"; + +import ChatPage from "@/components/chat/chat-page"; + +export default function Page({ params }: { params: { id: string } }) { + const [chatId, setChatId] = React.useState(""); + React.useEffect(() => { + if (params.id) { + setChatId(params.id); + } + }, [params.id]); + return ; +} diff --git a/src/app/favicon.ico b/src/app/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..718d6fea4835ec2d246af9800eddb7ffb276240c Binary files /dev/null and b/src/app/favicon.ico differ diff --git a/src/app/globals.css b/src/app/globals.css new file mode 100644 index 0000000000000000000000000000000000000000..78d22c3ba84d5ef443b19554fde7a45d0b0e464d --- /dev/null +++ b/src/app/globals.css @@ -0,0 +1,95 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + + +@layer base { + :root { + --background: 0 0% 100%; + --foreground: 240 10% 3.9%; + --card: 0 0% 100%; + --card-foreground: 240 10% 3.9%; + --popover: 0 0% 100%; + --popover-foreground: 240 10% 3.9%; + --primary: 240 5.9% 10%; + --primary-foreground: 0 0% 98%; + --secondary: 240 4.8% 95.9%; + --secondary-foreground: 240 5.9% 10%; + --muted: 240 4.8% 95.9%; + --muted-foreground: 240 3.8% 46.1%; + --accent: 240 4.8% 95.9%; + --accent-foreground: 240 5.9% 10%; + --destructive: 0 84.2% 60.2%; + --destructive-foreground: 0 0% 98%; + --border: 240 5.9% 90%; + --input: 240 5.9% 90%; + --ring: 240 5.9% 10%; + --radius: 1rem; + } + + .dark { + --background: 0 0% 9%; + --foreground: 0 0% 98%; + --card: 0 0% 12%; + --card-foreground: 0 0% 98%; + --popover: 0 0% 12%; + --popover-foreground: 0 0% 98%; + --primary: 0 0% 98%; + --primary-foreground: 240 5.9% 10%; + --secondary: 240 3.7% 15.9%; + --secondary-foreground: 0 0% 98%; + --muted: 240 3.7% 15.9%; + --muted-foreground: 240 5% 64.9%; + --accent: 240 3.7% 15.9%; + --accent-foreground: 0 0% 98%; + --destructive: 0 62.8% 30.6%; + --destructive-foreground: 0 0% 98%; + --border: 240 3.7% 15.9%; + --input: 240 3.7% 15.9%; + --ring: 240 4.9% 83.9%; + } +} + +@layer base { + * { + @apply border-border; + } + body { + @apply bg-background text-foreground; + } +} + +#scroller * { + overflow-anchor: none; +} + +#anchor { + overflow-anchor: auto; + height: 1px; +} + +:root { + --scrollbar-thumb-color: #ccc; + --scrollbar-thumb-hover-color: #aaa; +} + +::-webkit-scrollbar { + width: 6px; + height: 6px; +} + +::-webkit-scrollbar-thumb { + background-color: var(--scrollbar-thumb-color); + border-radius: 999px; + transition: width 0.3s, height 0.3s, visibility 0.3s; +} + +::-webkit-scrollbar-thumb:hover { + background-color: var(--scrollbar-thumb-hover-color); +} + +::-webkit-scrollbar-thumb:not(:hover) { + width: 0; + height: 0; + visibility: hidden; +} \ No newline at end of file diff --git a/src/app/layout.tsx b/src/app/layout.tsx new file mode 100644 index 0000000000000000000000000000000000000000..9a66df1ab2dc04f8c3ca0b7fde26e44f339e3e25 --- /dev/null +++ b/src/app/layout.tsx @@ -0,0 +1,36 @@ +import { ThemeProvider } from "@/providers/theme-provider"; +import type { Metadata } from "next"; + +import { Toaster } from "@/components/ui/sonner"; +import "./globals.css"; + +export const runtime = "edge"; // 'nodejs' (default) | 'edge' + +export const metadata: Metadata = { + title: "vLLM UI", + description: "vLLM chatbot web interface", +}; + +export const viewport = { + width: "device-width", + initialScale: 1, + maximumScale: 1, + userScalable: 1, +}; + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + + + {children} + + + + + ); +} diff --git a/src/app/page.tsx b/src/app/page.tsx new file mode 100644 index 0000000000000000000000000000000000000000..1da3b6c39a1e1d45506bae1e2e773404df14713c --- /dev/null +++ b/src/app/page.tsx @@ -0,0 +1,9 @@ +"use client"; + +import React from "react"; +import ChatPage from "@/components/chat/chat-page"; + +export default function Home() { + const [chatId, setChatId] = React.useState(""); + return ; +} diff --git a/src/components/chat/chat-bottombar.tsx b/src/components/chat/chat-bottombar.tsx new file mode 100644 index 0000000000000000000000000000000000000000..e65f825d72907c6de58422d33403ae1d2e47c884 --- /dev/null +++ b/src/components/chat/chat-bottombar.tsx @@ -0,0 +1,100 @@ +"use client"; + +import React from "react"; + +import { PaperPlaneIcon, StopIcon } from "@radix-ui/react-icons"; +import { ChatRequestOptions } from "ai"; +import mistralTokenizer from "mistral-tokenizer-js"; +import TextareaAutosize from "react-textarea-autosize"; + +import { tokenLimit } from "@/lib/token-counter"; +import { Button } from "../ui/button"; + +interface ChatBottombarProps { + selectedModel: string | undefined; + input: string; + handleInputChange: (e: React.ChangeEvent) => void; + handleSubmit: ( + e: React.FormEvent, + chatRequestOptions?: ChatRequestOptions + ) => void; + isLoading: boolean; + stop: () => void; +} + +export default function ChatBottombar({ + selectedModel, + input, + handleInputChange, + handleSubmit, + isLoading, + stop, +}: ChatBottombarProps) { + const inputRef = React.useRef(null); + const hasSelectedModel = selectedModel && selectedModel !== ""; + + const handleKeyPress = (e: React.KeyboardEvent) => { + if (e.key === "Enter" && !e.shiftKey && hasSelectedModel && !isLoading) { + e.preventDefault(); + handleSubmit(e as unknown as React.FormEvent); + } + }; + const tokenCount = input ? mistralTokenizer.encode(input).length - 1 : 0; + + return ( +
+
+ {/*
*/} +
+
+ +
+ {tokenCount > tokenLimit ? ( + + {tokenCount} token{tokenCount == 1 ? "" : "s"} + + ) : ( + + {tokenCount} token{tokenCount == 1 ? "" : "s"} + + )} +
+ {!isLoading ? ( + + ) : ( + + )} + +
+
+
+ Enter to send, Shift + Enter for new line +
+
+ ); +} diff --git a/src/components/chat/chat-layout.tsx b/src/components/chat/chat-layout.tsx new file mode 100644 index 0000000000000000000000000000000000000000..0a9ba5510c246e7145a36b590071b38fd4511cd4 --- /dev/null +++ b/src/components/chat/chat-layout.tsx @@ -0,0 +1,83 @@ +"use client"; + +import React, { useEffect, useState } from "react"; + +import { Sidebar } from "../sidebar"; +import Chat, { ChatProps, ChatTopbarProps } from "./chat"; + +interface ChatLayoutProps { + defaultLayout: number[] | undefined; + defaultCollapsed?: boolean; + navCollapsedSize: number; + chatId: string; +} + +type MergedProps = ChatLayoutProps & ChatProps & ChatTopbarProps; + +export function ChatLayout({ + defaultLayout = [30, 160], + defaultCollapsed = false, + navCollapsedSize = 768, + messages, + input, + handleInputChange, + handleSubmit, + isLoading, + error, + stop, + chatId, + setChatId, + chatOptions, + setChatOptions, +}: MergedProps) { + const [isCollapsed, setIsCollapsed] = React.useState(false); + const [isMobile, setIsMobile] = useState(false); + + useEffect(() => { + const checkScreenWidth = () => { + setIsMobile(window.innerWidth <= 768); + setIsCollapsed(window.innerWidth <= 768); + }; + + // Initial check + checkScreenWidth(); + + // Event listener for screen width changes + window.addEventListener("resize", checkScreenWidth); + + // Cleanup the event listener on component unmount + return () => { + window.removeEventListener("resize", checkScreenWidth); + }; + }, []); + + return ( +
+
+ +
+
+ +
+
+ ); +} diff --git a/src/components/chat/chat-list.tsx b/src/components/chat/chat-list.tsx new file mode 100644 index 0000000000000000000000000000000000000000..38544c957b568fbf6231c3083ef15c90e56abe86 --- /dev/null +++ b/src/components/chat/chat-list.tsx @@ -0,0 +1,136 @@ +import React, { useEffect, useRef } from "react"; + +import Image from "next/image"; + +import OllamaLogo from "../../../public/ollama.png"; +import CodeDisplayBlock from "../code-display-block"; +import { Message } from "ai"; + +interface ChatListProps { + messages: Message[]; + isLoading: boolean; +} + +const MessageToolbar = () => ( +
+
+ Regenerate + Edit +
+
+); + +export default function ChatList({ messages, isLoading }: ChatListProps) { + const bottomRef = useRef(null); + + const scrollToBottom = () => { + bottomRef.current?.scrollIntoView({ behavior: "auto", block: "end" }); + }; + + useEffect(() => { + if (isLoading) { + return; + } + // if user scrolls up, disable auto-scroll + scrollToBottom(); + }, [messages, isLoading]); + + if (messages.length === 0) { + return ( +
+
+ AI +

+ How can I help you today? +

+
+
+ ); + } + + return ( +
+
+ {messages + .filter((message) => message.role !== "system") + .map((message, index) => ( +
+
+
+
+
+ {message.role === "user" ? ( +
+ ) : ( + AI + )} +
+
+
+ {message.role === "user" && ( +
+
You
+
+ {message.content} +
+ +
+ )} + {message.role === "assistant" && ( +
+
Assistant
+
+ + {/* Check if the message content contains a code block */} + {message.content.split("```").map((part, index) => { + if (index % 2 === 0) { + return ( + + {part} + + ); + } else { + return ( + + ); + } + })} + {isLoading && + messages.indexOf(message) === messages.length - 1 && ( + + ... + + )} + +
+ +
+ )} +
+
+ ))} +
+
+
+ ); +} diff --git a/src/components/chat/chat-options.ts b/src/components/chat/chat-options.ts new file mode 100644 index 0000000000000000000000000000000000000000..380969636043b2d9e5cc667dfabd2847c4b409e7 --- /dev/null +++ b/src/components/chat/chat-options.ts @@ -0,0 +1,5 @@ +export interface ChatOptions { + selectedModel?: string; + systemPrompt?: string; + temperature?: number; +} diff --git a/src/components/chat/chat-page.tsx b/src/components/chat/chat-page.tsx new file mode 100644 index 0000000000000000000000000000000000000000..00096309d26944689705be3ad93cf8f1a9347837 --- /dev/null +++ b/src/components/chat/chat-page.tsx @@ -0,0 +1,110 @@ +"use client"; + +import React from "react"; + +import { ChatRequestOptions } from "ai"; +import { useChat } from "ai/react"; +import { toast } from "sonner"; +import useLocalStorageState from "use-local-storage-state"; +import { v4 as uuidv4 } from "uuid"; + +import { ChatLayout } from "@/components/chat/chat-layout"; +import { ChatOptions } from "@/components/chat/chat-options"; +import { basePath } from "@/lib/utils"; + +interface ChatPageProps { + chatId: string; + setChatId: React.Dispatch>; +} +export default function ChatPage({ chatId, setChatId }: ChatPageProps) { + const { + messages, + input, + handleInputChange, + handleSubmit, + isLoading, + error, + stop, + setMessages, + } = useChat({ + api: basePath + "/api/chat", + streamMode: "text", + onError: (error) => { + toast.error("Something went wrong: " + error); + }, + }); + const [chatOptions, setChatOptions] = useLocalStorageState( + "chatOptions", + { + defaultValue: { + selectedModel: "", + systemPrompt: "", + temperature: 0.9, + }, + } + ); + + React.useEffect(() => { + if (chatId) { + const item = localStorage.getItem(`chat_${chatId}`); + if (item) { + setMessages(JSON.parse(item)); + } + } else { + setMessages([]); + } + }, [setMessages, chatId]); + + React.useEffect(() => { + if (!isLoading && !error && chatId && messages.length > 0) { + // Save messages to local storage + localStorage.setItem(`chat_${chatId}`, JSON.stringify(messages)); + // Trigger the storage event to update the sidebar component + window.dispatchEvent(new Event("storage")); + } + }, [messages, chatId, isLoading, error]); + + const onSubmit = (e: React.FormEvent) => { + e.preventDefault(); + + if (messages.length === 0) { + // Generate a random id for the chat + const id = uuidv4(); + setChatId(id); + } + + setMessages([...messages]); + + // Prepare the options object with additional body data, to pass the model. + const requestOptions: ChatRequestOptions = { + options: { + body: { + chatOptions: chatOptions, + }, + }, + }; + + // Call the handleSubmit function with the options + handleSubmit(e, requestOptions); + }; + + return ( +
+ +
+ ); +} diff --git a/src/components/chat/chat-topbar.tsx b/src/components/chat/chat-topbar.tsx new file mode 100644 index 0000000000000000000000000000000000000000..eee845680533468d09c1d38a645884d7b478e783 --- /dev/null +++ b/src/components/chat/chat-topbar.tsx @@ -0,0 +1,175 @@ +"use client"; + +import React, { useEffect } from "react"; + +import { + CheckCircledIcon, + CrossCircledIcon, + DotFilledIcon, + HamburgerMenuIcon, + InfoCircledIcon, +} from "@radix-ui/react-icons"; +import { Message } from "ai/react"; +import { toast } from "sonner"; + +import { Sheet, SheetContent, SheetTrigger } from "@/components/ui/sheet"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { encodeChat, tokenLimit } from "@/lib/token-counter"; +import { basePath, useHasMounted } from "@/lib/utils"; +import { Sidebar } from "../sidebar"; +import { ChatOptions } from "./chat-options"; + +interface ChatTopbarProps { + chatOptions: ChatOptions; + setChatOptions: React.Dispatch>; + isLoading: boolean; + chatId?: string; + setChatId: React.Dispatch>; + messages: Message[]; +} + +export default function ChatTopbar({ + chatOptions, + setChatOptions, + isLoading, + chatId, + setChatId, + messages, +}: ChatTopbarProps) { + const hasMounted = useHasMounted(); + + const currentModel = chatOptions && chatOptions.selectedModel; + const [error, setError] = React.useState(undefined); + + const fetchData = async () => { + if (!hasMounted) { + return null; + } + try { + const res = await fetch(basePath + "/api/models"); + + if (!res.ok) { + const errorResponse = await res.json(); + const errorMessage = `Connection to vLLM server failed: ${errorResponse.error} [${res.status} ${res.statusText}]`; + throw new Error(errorMessage); + } + + const data = await res.json(); + // Extract the "name" field from each model object and store them in the state + const modelNames = data.data.map((model: any) => model.id); + // save the first and only model in the list as selectedModel in localstorage + setChatOptions({ ...chatOptions, selectedModel: modelNames[0] }); + } catch (error) { + setChatOptions({ ...chatOptions, selectedModel: undefined }); + toast.error(error as string); + } + }; + + useEffect(() => { + fetchData(); + }, [hasMounted]); + + if (!hasMounted) { + return ( +
+ + Booting up.. +
+ ); + } + + const chatTokens = messages.length > 0 ? encodeChat(messages) : 0; + + return ( +
+ + +
+ +
+ + +
+ +
+ + + +
+
+ {currentModel !== undefined && ( + <> + {isLoading ? ( + + ) : ( + + + + + + + + +

Current Model

+

{currentModel}

+ + + + )} + + {isLoading ? "Generating.." : "Ready"} + + + )} + {currentModel === undefined && ( + <> + + Connection to vLLM server failed + + )} +
+
+ {chatTokens > tokenLimit && ( + + + + + + + + +

+ Token limit exceeded. Truncating middle messages. +

+ + + + )} + {messages.length > 0 && ( + + {chatTokens} / {tokenLimit} token{chatTokens > 1 ? "s" : ""} + + )} +
+
+
+ ); +} diff --git a/src/components/chat/chat.tsx b/src/components/chat/chat.tsx new file mode 100644 index 0000000000000000000000000000000000000000..b16b13aa737a5ca8d6b8490d7d7ebf1ea4f99802 --- /dev/null +++ b/src/components/chat/chat.tsx @@ -0,0 +1,70 @@ +import React from "react"; + +import { ChatRequestOptions } from "ai"; +import { Message } from "ai/react"; + +import ChatBottombar from "./chat-bottombar"; +import ChatList from "./chat-list"; +import { ChatOptions } from "./chat-options"; +import ChatTopbar from "./chat-topbar"; + +export interface ChatProps { + chatId?: string; + setChatId: React.Dispatch>; + messages: Message[]; + input: string; + handleInputChange: (e: React.ChangeEvent) => void; + handleSubmit: ( + e: React.FormEvent, + chatRequestOptions?: ChatRequestOptions + ) => void; + isLoading: boolean; + error: undefined | Error; + stop: () => void; +} + +export interface ChatTopbarProps { + chatOptions: ChatOptions; + setChatOptions: React.Dispatch>; +} + +export default function Chat({ + messages, + input, + handleInputChange, + handleSubmit, + isLoading, + error, + stop, + chatOptions, + setChatOptions, + chatId, + setChatId, +}: ChatProps & ChatTopbarProps) { + return ( +
+ + + + + +
+ ); +} diff --git a/src/components/code-display-block.tsx b/src/components/code-display-block.tsx new file mode 100644 index 0000000000000000000000000000000000000000..b5a1dc19b6296c6835fab8381173043c0c365968 --- /dev/null +++ b/src/components/code-display-block.tsx @@ -0,0 +1,56 @@ +"use client"; +import React from "react"; + +import { CheckIcon, CopyIcon } from "@radix-ui/react-icons"; +import { useTheme } from "next-themes"; +import { CodeBlock, dracula, github } from "react-code-blocks"; +import { toast } from "sonner"; + +import { Button } from "./ui/button"; + +interface ButtonCodeblockProps { + code: string; + lang: string; +} + +export default function CodeDisplayBlock({ code, lang }: ButtonCodeblockProps) { + const [isCopied, setisCopied] = React.useState(false); + const { theme } = useTheme(); + + const copyToClipboard = () => { + navigator.clipboard.writeText(code); + setisCopied(true); + toast.success("Code copied to clipboard!"); + setTimeout(() => { + setisCopied(false); + }, 1500); + }; + + return ( +
+ + +
+ ); +} diff --git a/src/components/settings-clear-chats.tsx b/src/components/settings-clear-chats.tsx new file mode 100644 index 0000000000000000000000000000000000000000..0ce88490b073bdfe63dba8882ed37764635e25f3 --- /dev/null +++ b/src/components/settings-clear-chats.tsx @@ -0,0 +1,70 @@ +"use client"; + +import * as React from "react"; + +import { + Dialog, + DialogClose, + DialogContent, + DialogDescription, + DialogTrigger, +} from "@radix-ui/react-dialog"; +import { TrashIcon } from "@radix-ui/react-icons"; +import { useRouter } from "next/navigation"; + +import { useHasMounted } from "@/lib/utils"; +import { DialogHeader } from "./ui/dialog"; + +export default function ClearChatsButton() { + const hasMounted = useHasMounted(); + const router = useRouter(); + + if (!hasMounted) { + return null; + } + + const chats = Object.keys(localStorage).filter((key) => + key.startsWith("chat_") + ); + + const disabled = chats.length === 0; + + const clearChats = () => { + chats.forEach((key) => { + localStorage.removeItem(key); + }); + window.dispatchEvent(new Event("storage")); + router.push("/"); + }; + + return ( + + + + Clear chats + + + + + Are you sure you want to delete all chats? This action cannot be + undone. + +
+ + Cancel + + clearChats()} + > + Delete + +
+ + + + ); +} diff --git a/src/components/settings-theme-toggle.tsx b/src/components/settings-theme-toggle.tsx new file mode 100644 index 0000000000000000000000000000000000000000..fe2218f3f4fb894748698334c4e03ece1958ecc6 --- /dev/null +++ b/src/components/settings-theme-toggle.tsx @@ -0,0 +1,36 @@ +"use client"; + +import * as React from "react"; + +import { MoonIcon, SunIcon } from "@radix-ui/react-icons"; +import { useTheme } from "next-themes"; + +import { useHasMounted } from "@/lib/utils"; +import { Button } from "./ui/button"; + +export default function SettingsThemeToggle() { + const hasMounted = useHasMounted(); + const { setTheme, theme } = useTheme(); + + if (!hasMounted) { + return null; + } + + const nextTheme = theme === "light" ? "dark" : "light"; + + return ( + + ); +} diff --git a/src/components/settings.tsx b/src/components/settings.tsx new file mode 100644 index 0000000000000000000000000000000000000000..90483eb4817fe5661d22f1de8a3fc9576a802ca4 --- /dev/null +++ b/src/components/settings.tsx @@ -0,0 +1,71 @@ +"use client"; + +import ClearChatsButton from "./settings-clear-chats"; +import SettingsThemeToggle from "./settings-theme-toggle"; +import SystemPrompt, { SystemPromptProps } from "./system-prompt"; +import { Input } from "./ui/input"; + +const TemperatureSlider = ({ + chatOptions, + setChatOptions, +}: SystemPromptProps) => { + const handleTemperatureChange = (e: React.ChangeEvent) => { + setChatOptions({ ...chatOptions, temperature: parseFloat(e.target.value) }); + }; + + return ( +
+
+ + +
+ +
+ +
+
+ ); +}; + +export default function Settings({ + chatOptions, + setChatOptions, +}: SystemPromptProps) { + return ( + <> + + + + + + ); +} diff --git a/src/components/sidebar-skeleton.tsx b/src/components/sidebar-skeleton.tsx new file mode 100644 index 0000000000000000000000000000000000000000..97a1d510fed5065acb39584e24fe18b9213fe1c1 --- /dev/null +++ b/src/components/sidebar-skeleton.tsx @@ -0,0 +1,33 @@ +import { Skeleton } from "@/components/ui/skeleton"; + +export default function SidebarSkeleton() { + return ( +
+
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ ); +} diff --git a/src/components/sidebar-tabs.tsx b/src/components/sidebar-tabs.tsx new file mode 100644 index 0000000000000000000000000000000000000000..6447476373e5370ea0c4d14aca3b4f2615f2742f --- /dev/null +++ b/src/components/sidebar-tabs.tsx @@ -0,0 +1,167 @@ +"use client"; + +import React from "react"; + +import { DialogClose } from "@radix-ui/react-dialog"; +import { ChatBubbleIcon, GearIcon, TrashIcon } from "@radix-ui/react-icons"; +import * as Tabs from "@radix-ui/react-tabs"; +import { Message } from "ai/react"; +import Link from "next/link"; + +import { buttonVariants } from "@/components/ui/button"; +import { cn } from "@/lib/utils"; +import { ChatOptions } from "./chat/chat-options"; +import Settings from "./settings"; +import SidebarSkeleton from "./sidebar-skeleton"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "./ui/dialog"; + +interface Chats { + [key: string]: { chatId: string; messages: Message[] }[]; +} +interface SidebarTabsProps { + isLoading: boolean; + localChats: Chats; + selectedChatId: string; + chatOptions: ChatOptions; + setChatOptions: React.Dispatch>; + handleDeleteChat: (chatId: string) => void; +} + +const SidebarTabs = ({ + localChats, + selectedChatId, + isLoading, + chatOptions, + setChatOptions, + handleDeleteChat, +}: SidebarTabsProps) => ( + +
+ +
+ {Object.keys(localChats).length > 0 && ( + <> + {Object.keys(localChats).map((group, index) => ( +
+

+ {group} +

+
    + {localChats[group].map( + ({ chatId, messages }, chatIndex) => { + const isSelected = + chatId.substring(5) === selectedChatId; + return ( +
  1. +
    + + + {messages.length > 0 + ? messages[0].content + : ""} + + +
    +
    + + + + + + + Delete chat? + + Are you sure you want to delete this chat? + This action cannot be undone. + +
    + + Cancel + + + handleDeleteChat(chatId)} + > + Delete + +
    +     +     +     +
    +
    2. + ); + } + )} +
+
+ ))} + + )} + {isLoading && } +
+ + +
+ +
+ +
+
+ + + + {/* Chats */} + + + + {/* Settings */} + + +
+ +); + +export default SidebarTabs; diff --git a/src/components/sidebar.tsx b/src/components/sidebar.tsx new file mode 100644 index 0000000000000000000000000000000000000000..e32cbe274a2599cb525e9e7193e5b99db98ee618 --- /dev/null +++ b/src/components/sidebar.tsx @@ -0,0 +1,163 @@ +"use client"; +import { useEffect, useState } from "react"; + +import { Pencil2Icon } from "@radix-ui/react-icons"; +import { Message } from "ai/react"; +import Image from "next/image"; + +import OllamaLogo from "../../public/ollama.png"; +import { ChatOptions } from "./chat/chat-options"; +import SidebarTabs from "./sidebar-tabs"; +import Link from "next/link"; + +interface SidebarProps { + isCollapsed: boolean; + onClick?: () => void; + isMobile: boolean; + chatId: string; + setChatId: React.Dispatch>; + chatOptions: ChatOptions; + setChatOptions: React.Dispatch>; +} + +interface Chats { + [key: string]: { chatId: string; messages: Message[] }[]; +} + +export function Sidebar({ + isCollapsed, + isMobile, + chatId, + setChatId, + chatOptions, + setChatOptions, +}: SidebarProps) { + const [localChats, setLocalChats] = useState({}); + const [isLoading, setIsLoading] = useState(true); + + useEffect(() => { + setLocalChats(getLocalstorageChats()); + const handleStorageChange = () => { + setLocalChats(getLocalstorageChats()); + }; + window.addEventListener("storage", handleStorageChange); + return () => { + window.removeEventListener("storage", handleStorageChange); + }; + }, [chatId]); + + const getLocalstorageChats = (): Chats => { + const chats = Object.keys(localStorage).filter((key) => + key.startsWith("chat_") + ); + + if (chats.length === 0) { + setIsLoading(false); + } + + // Map through the chats and return an object with chatId and messages + const chatObjects = chats.map((chat) => { + const item = localStorage.getItem(chat); + return item + ? { chatId: chat, messages: JSON.parse(item) } + : { chatId: "", messages: [] }; + }); + + // Sort chats by the createdAt date of the first message of each chat + chatObjects.sort((a, b) => { + const aDate = new Date(a.messages[0].createdAt); + const bDate = new Date(b.messages[0].createdAt); + return bDate.getTime() - aDate.getTime(); + }); + + const groupChatsByDate = ( + chats: { chatId: string; messages: Message[] }[] + ) => { + const today = new Date(); + const yesterday = new Date(today); + yesterday.setDate(yesterday.getDate() - 1); + + const groupedChats: Chats = {}; + + chats.forEach((chat) => { + const createdAt = new Date(chat.messages[0].createdAt ?? ""); + const diffInDays = Math.floor( + (today.getTime() - createdAt.getTime()) / (1000 * 3600 * 24) + ); + + let group: string; + if (diffInDays === 0) { + group = "Today"; + } else if (diffInDays === 1) { + group = "Yesterday"; + } else if (diffInDays <= 7) { + group = "Previous 7 Days"; + } else if (diffInDays <= 30) { + group = "Previous 30 Days"; + } else { + group = "Older"; + } + + if (!groupedChats[group]) { + groupedChats[group] = []; + } + groupedChats[group].push(chat); + }); + + return groupedChats; + }; + + setIsLoading(false); + const groupedChats = groupChatsByDate(chatObjects); + + return groupedChats; + }; + + const handleDeleteChat = (chatId: string) => { + localStorage.removeItem(chatId); + setLocalChats(getLocalstorageChats()); + }; + + return ( +
+
+ { + setChatId(""); + }} + > +
+
+ {!isCollapsed && !isMobile && ( + AI + )} + New chat +
+ +
+ +
+ +
+ ); +} diff --git a/src/components/system-prompt.tsx b/src/components/system-prompt.tsx new file mode 100644 index 0000000000000000000000000000000000000000..c3fd075bbe59ed222ad2054b796a1dd63be1e7bd --- /dev/null +++ b/src/components/system-prompt.tsx @@ -0,0 +1,55 @@ +"use client"; + +import { Dispatch, SetStateAction, useEffect, useState } from "react"; + +import { toast } from "sonner"; +import { useDebounce } from "use-debounce"; + +import { useHasMounted } from "@/lib/utils"; +import { ChatOptions } from "./chat/chat-options"; +import { Textarea } from "./ui/textarea"; + +export interface SystemPromptProps { + chatOptions: ChatOptions; + setChatOptions: Dispatch>; +} +export default function SystemPrompt({ + chatOptions, + setChatOptions, +}: SystemPromptProps) { + const hasMounted = useHasMounted(); + + const systemPrompt = chatOptions ? chatOptions.systemPrompt : ""; + const [text, setText] = useState(systemPrompt || ""); + const [debouncedText] = useDebounce(text, 500); + + useEffect(() => { + if (!hasMounted) { + return; + } + if (debouncedText !== systemPrompt) { + setChatOptions({ ...chatOptions, systemPrompt: debouncedText }); + toast.success("System prompt updated", { duration: 1000 }); + } + }, [hasMounted, debouncedText]); + + return ( +
+
+

System prompt

+
+ +
+