Initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+.parcel-cache

README.md

@@ -0,0 +1,25 @@
+---
+title: "Web-LLM: Mistral 7B OpenOrca"
+emoji: 🌊
+colorFrom: pink
+colorTo: purple
+sdk: static
+app_file: dist/index.html
+pinned: false
+license: apache-2.0
+models:
+    - Felladrin/mlc-chat-Mistral-7B-OpenOrca-q4f32_1
+    - Open-Orca/Mistral-7B-OpenOrca
+---
+
+# Web-LLM: Mistral 7B OpenOrca
+
+Demo Space for [Felladrin/mlc-chat-Mistral-7B-OpenOrca-q4f32_1](https://huggingface.co/Felladrin/mlc-chat-Mistral-7B-OpenOrca-q4f32_1) model.
+
+## Contributing
+
+Before start developing or building, run `npm ci` to install the dependencies.
+
+To run the app locally for development, run `npm start`.
+
+To build the app to `dist` folder, run `npm run build`.

dist/index.9744ff88.css

@@ -0,0 +1 @@
+.chatui{border:2px solid #ddd;border-radius:5px;flex-flow:column wrap;justify-content:space-between;width:100%;max-width:867px;height:600px;margin:25px 10px;display:flex;box-shadow:0 15px 15px -5px #0003}s .chatui-header{color:#666;background:#eee;border-bottom:2px solid #ddd;justify-content:space-between;padding:10px;display:flex}.chatui-chat{flex:1;padding:10px;overflow-y:auto}.chatui-chat::-webkit-scrollbar{width:6px}.chatui-chat::-webkit-scrollbar-track{background:#ddd}.chatui-chat::-webkit-scrollbar-thumb{background:#bdbdbd}.msg{align-items:flex-end;margin-bottom:10px;display:flex}.msg:last-of-type{margin:0}.msg-bubble{background:#ececec;border-radius:15px;max-width:450px;padding:15px}.left-msg .msg-bubble{border-bottom-left-radius:0}.error-msg .msg-bubble{color:#f15959;border-bottom-left-radius:0}.init-msg .msg-bubble{border-bottom-left-radius:0}.right-msg{flex-direction:row-reverse}.right-msg .msg-bubble{color:#fff;background:#579ffb;border-bottom-right-radius:0}.chatui-inputarea{background:#eee;border-top:2px solid #ddd;padding:10px;display:flex}.chatui-inputarea *{border:none;border-radius:3px;padding:10px;font-size:1em}.chatui-input{background:#ddd;flex:1}.chatui-reset-btn{cursor:pointer;background:#ececec;border-radius:8px;width:200px;margin-left:10px;font-weight:700}.chatui-reset-btn:hover{background:#dcdada}.chatui-send-btn{color:#fff;cursor:pointer;background:#579ffb;margin-left:10px;font-weight:700}.chatui-send-btn:hover{background:#577bfb}.chatui-chat{background-color:#fcfcfe}

dist/index.html

@@ -0,0 +1 @@
+<script type="module" src="index.runtime.2846421e.js"></script><select id="chatui-select" style="display:none"></select><link href="index.9744ff88.css" rel="stylesheet" type="text/css"><div class="chatui"> <div class="chatui-chat" id="chatui-chat" height="100"></div> <div class="chatui-inputarea"> <input id="chatui-input" type="text" class="chatui-input" placeholder="Enter your message..."> <button id="chatui-send-btn" class="chatui-send-btn">Send</button> </div> </div><div class="chatui-extra-control"> <button id="chatui-reset-btn" class="chatui-reset-btn">Reset</button> <label id="chatui-info-label"></label> </div><script type="module" src="index.bd4d70df.js"></script>

dist/index.runtime.2846421e.js

@@ -0,0 +1 @@
+var e=globalThis,r={},t={},a=e.parcelRequireba71;null==a&&((a=function(e){if(e in r)return r[e].exports;if(e in t){var a=t[e];delete t[e];var n={id:e,exports:{}};return r[e]=n,a.call(n.exports,n,n.exports),n.exports}var o=Error("Cannot find module '"+e+"'");throw o.code="MODULE_NOT_FOUND",o}).register=function(e,r){t[e]=r},e.parcelRequireba71=a),(0,a.register)("dRo73",function(e,r){Object.defineProperty(e.exports,"register",{get:()=>t,set:e=>t=e,enumerable:!0,configurable:!0});var t,a=new Map;t=function(e,r){for(var t=0;t<r.length-1;t+=2)a.set(r[t],{baseUrl:e,path:r[t+1]})}}),a("dRo73").register(new URL("",import.meta.url).toString(),JSON.parse('["lvtVP","index.bd4d70df.js","ayGjr","worker.6af90c76.js"]'));

package.json

@@ -0,0 +1,19 @@
+{
+  "name": "simple-chat",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "start": "parcel src/index.html",
+    "build": "parcel build src/index.html --public-url ./ --no-source-maps"
+  },
+  "devDependencies": {
+    "buffer": "^6.0.3",
+    "parcel": "^2.8.3",
+    "process": "^0.11.10",
+    "typescript": "^5.2.2",
+    "url": "^0.11.3"
+  },
+  "dependencies": {
+    "@mlc-ai/web-llm": "^0.2.8"
+  }
+}

src/index.css

@@ -0,0 +1,127 @@
+ .chatui {
+   display: flex;
+   flex-flow: column wrap;
+   justify-content: space-between;
+   width: 100%;
+   max-width: 867px;
+   margin: 25px 10px;
+   height: 600px;
+   border: 2px solid #ddd;
+   border-radius: 5px;
+   box-shadow: 0 15px 15px -5px rgba(0, 0, 0, 0.2);
+ }
+
+ s .chatui-header {
+   display: flex;
+   justify-content: space-between;
+   padding: 10px;
+   border-bottom: 2px solid #ddd;
+   background: #eee;
+   color: #666;
+ }
+
+ .chatui-chat {
+   flex: 1;
+   overflow-y: auto;
+   padding: 10px;
+ }
+
+ .chatui-chat::-webkit-scrollbar {
+   width: 6px;
+ }
+
+ .chatui-chat::-webkit-scrollbar-track {
+   background: #ddd;
+ }
+
+ .chatui-chat::-webkit-scrollbar-thumb {
+   background: #bdbdbd;
+ }
+
+ .msg {
+   display: flex;
+   align-items: flex-end;
+   margin-bottom: 10px;
+ }
+
+ .msg:last-of-type {
+   margin: 0;
+ }
+
+ .msg-bubble {
+   max-width: 450px;
+   padding: 15px;
+   border-radius: 15px;
+   background: #ececec;
+ }
+
+ .left-msg .msg-bubble {
+   border-bottom-left-radius: 0;
+ }
+
+ .error-msg .msg-bubble {
+  border-bottom-left-radius: 0;
+  color: #f15959;
+}
+
+.init-msg .msg-bubble {
+  border-bottom-left-radius: 0;
+}
+
+ .right-msg {
+   flex-direction: row-reverse;
+ }
+
+ .right-msg .msg-bubble {
+   background: #579ffb;
+   color: #fff;
+   border-bottom-right-radius: 0;
+ }
+
+ .chatui-inputarea {
+   display: flex;
+   padding: 10px;
+   border-top: 2px solid #ddd;
+   background: #eee;
+ }
+
+ .chatui-inputarea * {
+   padding: 10px;
+   border: none;
+   border-radius: 3px;
+   font-size: 1em;
+ }
+
+ .chatui-input {
+   flex: 1;
+   background: #ddd;
+ }
+
+ .chatui-reset-btn {
+   margin-left: 10px;
+   background: #ececec;
+   font-weight: bold;
+   border-radius: 8px;
+   width: 200px;
+   cursor: pointer;
+ }
+
+ .chatui-reset-btn:hover {
+  background: #dcdada;
+}
+
+ .chatui-send-btn {
+  margin-left: 10px;
+  background: #579ffb;
+  color: #fff;
+  font-weight: bold;
+  cursor: pointer;
+}
+
+.chatui-send-btn:hover {
+  background: #577bfb;
+}
+
+ .chatui-chat {
+   background-color: #fcfcfe;
+ }

src/index.html

@@ -0,0 +1,24 @@
+<select id="chatui-select" style="display: none"></select>
+
+<link href="./index.css" rel="stylesheet" type="text/css" />
+
+<div class="chatui">
+  <div class="chatui-chat" id="chatui-chat" height="100"></div>
+
+  <div class="chatui-inputarea">
+    <input
+      id="chatui-input"
+      type="text"
+      class="chatui-input"
+      placeholder="Enter your message..."
+    />
+    <button id="chatui-send-btn" class="chatui-send-btn">Send</button>
+  </div>
+</div>
+
+<div class="chatui-extra-control">
+  <button id="chatui-reset-btn" class="chatui-reset-btn">Reset</button>
+  <label id="chatui-info-label"></label>
+</div>
+<!--- Place script after ui to make sure ui loads first -->
+<script type="module" src="./index.ts"></script>

src/index.ts

@@ -0,0 +1,279 @@
+import {
+  ChatInterface,
+  ChatModule,
+  ChatRestModule,
+  ChatWorkerClient,
+} from "@mlc-ai/web-llm";
+
+function getElementAndCheck(id: string): HTMLElement {
+  const element = document.getElementById(id);
+  if (element == null) {
+    throw Error("Cannot find element " + id);
+  }
+  return element;
+}
+
+const appConfig = {
+  model_list: [
+    {
+      model_url:
+        "https://huggingface.co/Felladrin/mlc-chat-Mistral-7B-OpenOrca-q4f32_1/resolve/main/params/",
+      local_id: "Mistral-7B-OpenOrca-q4f32_1",
+    },
+  ],
+  model_lib_map: {
+    "Mistral-7B-OpenOrca-q4f32_1":
+      "https://huggingface.co/Felladrin/mlc-chat-Mistral-7B-OpenOrca-q4f32_1/resolve/main/Mistral-7B-OpenOrca-q4f32_1-webgpu.wasm",
+  },
+  use_web_worker: true,
+};
+
+class ChatUI {
+  private uiChat: HTMLElement;
+  private uiChatInput: HTMLInputElement;
+  private uiChatInfoLabel: HTMLLabelElement;
+  private chat: ChatInterface;
+  private localChat: ChatInterface;
+  private config = appConfig;
+  private selectedModel: string;
+  private chatLoaded = false;
+  private requestInProgress = false;
+  // We use a request chain to ensure that
+  // all requests send to chat are sequentialized
+  private chatRequestChain: Promise<void> = Promise.resolve();
+
+  constructor(chat: ChatInterface, localChat: ChatInterface) {
+    // use web worker to run chat generation in background
+    this.chat = chat;
+    this.localChat = localChat;
+    // get the elements
+    this.uiChat = getElementAndCheck("chatui-chat");
+    this.uiChatInput = getElementAndCheck("chatui-input") as HTMLInputElement;
+    this.uiChatInfoLabel = getElementAndCheck(
+      "chatui-info-label"
+    ) as HTMLLabelElement;
+    // register event handlers
+    getElementAndCheck("chatui-reset-btn").onclick = () => {
+      this.onReset();
+    };
+    getElementAndCheck("chatui-send-btn").onclick = () => {
+      this.onGenerate();
+    };
+    // TODO: find other alternative triggers
+    getElementAndCheck("chatui-input").onkeypress = (event) => {
+      if (event.keyCode === 13) {
+        this.onGenerate();
+      }
+    };
+
+    const modelSelector = getElementAndCheck(
+      "chatui-select"
+    ) as HTMLSelectElement;
+    for (let i = 0; i < this.config.model_list.length; ++i) {
+      const item = this.config.model_list[i];
+      const opt = document.createElement("option");
+      opt.value = item.local_id;
+      opt.innerHTML = item.local_id;
+      opt.selected = i == 0;
+      modelSelector.appendChild(opt);
+    }
+    // Append local server option to the model selector
+    const localServerOpt = document.createElement("option");
+    localServerOpt.value = "Local Server";
+    localServerOpt.innerHTML = "Local Server";
+    modelSelector.append(localServerOpt);
+    this.selectedModel = modelSelector.value;
+    modelSelector.onchange = () => {
+      this.onSelectChange(modelSelector);
+    };
+  }
+  /**
+   * Push a task to the execution queue.
+   *
+   * @param task The task to be executed;
+   */
+  private pushTask(task: () => Promise<void>) {
+    const lastEvent = this.chatRequestChain;
+    this.chatRequestChain = lastEvent.then(task);
+  }
+  // Event handlers
+  // all event handler pushes the tasks to a queue
+  // that get executed sequentially
+  // the tasks previous tasks, which causes them to early stop
+  // can be interrupted by chat.interruptGenerate
+  private async onGenerate() {
+    if (this.requestInProgress) {
+      return;
+    }
+    this.pushTask(async () => {
+      await this.asyncGenerate();
+    });
+  }
+
+  private async onSelectChange(modelSelector: HTMLSelectElement) {
+    if (this.requestInProgress) {
+      // interrupt previous generation if any
+      this.chat.interruptGenerate();
+    }
+    // try reset after previous requests finishes
+    this.pushTask(async () => {
+      await this.chat.resetChat();
+      this.resetChatHistory();
+      await this.unloadChat();
+      this.selectedModel = modelSelector.value;
+      await this.asyncInitChat();
+    });
+  }
+
+  private async onReset() {
+    if (this.requestInProgress) {
+      // interrupt previous generation if any
+      this.chat.interruptGenerate();
+    }
+    // try reset after previous requests finishes
+    this.pushTask(async () => {
+      await this.chat.resetChat();
+      this.resetChatHistory();
+    });
+  }
+
+  // Internal helper functions
+  private appendMessage(kind, text) {
+    if (kind == "init") {
+      text = "[System Initalize] " + text;
+    }
+    if (this.uiChat === undefined) {
+      throw Error("cannot find ui chat");
+    }
+    const msg = `
+      <div class="msg ${kind}-msg">
+        <div class="msg-bubble">
+          <div class="msg-text">${text}</div>
+        </div>
+      </div>
+    `;
+    this.uiChat.insertAdjacentHTML("beforeend", msg);
+    this.uiChat.scrollTo(0, this.uiChat.scrollHeight);
+  }
+
+  private updateLastMessage(kind, text) {
+    if (kind == "init") {
+      text = "[System Initalize] " + text;
+    }
+    if (this.uiChat === undefined) {
+      throw Error("cannot find ui chat");
+    }
+    const matches = this.uiChat.getElementsByClassName(`msg ${kind}-msg`);
+    if (matches.length == 0) throw Error(`${kind} message do not exist`);
+    const msg = matches[matches.length - 1];
+    const msgText = msg.getElementsByClassName("msg-text");
+    if (msgText.length != 1) throw Error("Expect msg-text");
+    if (msgText[0].innerHTML == text) return;
+    const list = text.split("\n").map((t) => {
+      const item = document.createElement("div");
+      item.textContent = t;
+      return item;
+    });
+    msgText[0].innerHTML = "";
+    list.forEach((item) => msgText[0].append(item));
+    this.uiChat.scrollTo(0, this.uiChat.scrollHeight);
+  }
+
+  private resetChatHistory() {
+    const clearTags = ["left", "right", "init", "error"];
+    for (const tag of clearTags) {
+      // need to unpack to list so the iterator don't get affected by mutation
+      const matches = [...this.uiChat.getElementsByClassName(`msg ${tag}-msg`)];
+      for (const item of matches) {
+        this.uiChat.removeChild(item);
+      }
+    }
+    if (this.uiChatInfoLabel !== undefined) {
+      this.uiChatInfoLabel.innerHTML = "";
+    }
+  }
+
+  private async asyncInitChat() {
+    if (this.chatLoaded) return;
+    this.requestInProgress = true;
+    this.appendMessage("init", "");
+    const initProgressCallback = (report) => {
+      this.updateLastMessage("init", report.text);
+    };
+    this.chat.setInitProgressCallback(initProgressCallback);
+
+    try {
+      if (this.selectedModel != "Local Server") {
+        await this.chat.reload(this.selectedModel, undefined, this.config);
+      }
+    } catch (err) {
+      this.appendMessage("error", "Init error, " + err.toString());
+      console.log(err.stack);
+      this.unloadChat();
+      this.requestInProgress = false;
+      return;
+    }
+    this.requestInProgress = false;
+    this.chatLoaded = true;
+  }
+
+  private async unloadChat() {
+    await this.chat.unload();
+    this.chatLoaded = false;
+  }
+
+  /**
+   * Run generate
+   */
+  private async asyncGenerate() {
+    await this.asyncInitChat();
+    this.requestInProgress = true;
+    const prompt = this.uiChatInput.value;
+    if (prompt == "") {
+      this.requestInProgress = false;
+      return;
+    }
+
+    this.appendMessage("right", prompt);
+    this.uiChatInput.value = "";
+    this.uiChatInput.setAttribute("placeholder", "Generating...");
+
+    this.appendMessage("left", "");
+    const callbackUpdateResponse = (step, msg) => {
+      if (msg.length === 0) return this.chat.interruptGenerate();
+      this.updateLastMessage("left", msg);
+    };
+
+    try {
+      if (this.selectedModel == "Local Server") {
+        await this.localChat.generate(prompt, callbackUpdateResponse);
+        this.uiChatInfoLabel.innerHTML =
+          await this.localChat.runtimeStatsText();
+      } else {
+        await this.chat.generate(prompt, callbackUpdateResponse);
+        this.uiChatInfoLabel.innerHTML = await this.chat.runtimeStatsText();
+      }
+    } catch (err) {
+      this.appendMessage("error", "Generate error, " + err.toString());
+      console.log(err.stack);
+      await this.unloadChat();
+    }
+    this.uiChatInput.setAttribute("placeholder", "Enter your message...");
+    this.requestInProgress = false;
+  }
+}
+
+const useWebWorker = appConfig.use_web_worker;
+let chat: ChatInterface;
+let localChat: ChatInterface;
+
+if (useWebWorker) {
+  chat = new ChatWorkerClient(
+    new Worker(new URL("./worker.ts", import.meta.url), { type: "module" })
+  );
+  localChat = new ChatRestModule();
+} else {
+  chat = new ChatModule();
+  localChat = new ChatRestModule();
+}
+new ChatUI(chat, localChat);

src/worker.ts

@@ -0,0 +1,8 @@
+// Serve the chat workload through web worker
+import { ChatWorkerHandler, ChatModule } from "@mlc-ai/web-llm";
+
+const chat = new ChatModule();
+const handler = new ChatWorkerHandler(chat);
+self.onmessage = (msg: MessageEvent) => {
+  handler.onmessage(msg);
+};