import { env, pipeline } from './transformers/transformers.js'; const clipboardIcon = ` ` marked.use({ mangle: false, headerIds: false }); function log(i) { console.log(i); document.getElementById('status').innerText += `\n${i}`; } const sendButton = document.getElementById('send-button'); // adjusts the padding at the bottom of scrollWrapper to be the height of the input box function adjustPadding() { const inputBoxHeight = document.getElementById('input-area').offsetHeight; const scrollWrapper = document.getElementById('scroll-wrapper'); scrollWrapper.style.paddingBottom = `${inputBoxHeight + 15}px`; } // sets up padding resize whenever input box has its height changed const autoResizePadding = new ResizeObserver(() => { adjustPadding(); }); autoResizePadding.observe(document.getElementById('input-area')); // variables to handle auto-scroll // we only need one ResizeObserver and isAutoScrollOn variable globally // no need to make a new one for every time submitRequest is called const scrollWrapper = document.getElementById('scroll-wrapper'); let isAutoScrollOn = true; // autoscroll when new line is added const autoScroller = new ResizeObserver(() => { if (isAutoScrollOn) { scrollWrapper.scrollIntoView({ behavior: "smooth", block: "end" }); } }); // event listener for scrolling let lastKnownScrollPosition = 0; let ticking = false; document.addEventListener("scroll", (event) => { // if user has scrolled up and autoScroll is on we turn it off if (!ticking && isAutoScrollOn && window.scrollY < lastKnownScrollPosition) { window.requestAnimationFrame(() => { isAutoScrollOn = false; ticking = false; }); ticking = true; } // if user has scrolled nearly all the way down and autoScroll is disabled, re-enable else if (!ticking && !isAutoScrollOn && window.scrollY > lastKnownScrollPosition && // make sure scroll direction is down window.scrollY >= document.documentElement.scrollHeight - window.innerHeight - 30 // add 30px of space--no need to scroll all the way down, just most of the way ) { window.requestAnimationFrame(() => { isAutoScrollOn = true; ticking = false; }); ticking = true; } lastKnownScrollPosition = window.scrollY; }); function copyTextToClipboard(responseDiv, with_button) { let elem = responseDiv; if (with_button) { let copyButton = document.createElement('button'); copyButton.className = 'btn btn-secondary copy-button'; copyButton.innerHTML = clipboardIcon; elem = copyButton; } elem.onclick = () => { let text = responseDiv.hidden_text; if (!text) { text = responseDiv.innerText; } navigator.clipboard.writeText(text).then(() => { console.log('Text copied to clipboard'); }).catch(err => { console.error('Failed to copy text:', err); }); }; if (with_button) { responseDiv.appendChild(elem); } } let interrupt; // Function to handle the user input and call the API functions async function submitRequest() { if (sendButton.innerHTML == "Stop" && interrupt) { console.log("Stop"); interrupt.abort('Stop button pressed'); return; } document.getElementById('chat-container').style.display = 'block'; const input = document.getElementById('user-input').value; if (input.length == 0) { document.getElementById('chat-history').context = ""; let chatHistory = document.getElementById('chat-history'); while (chatHistory.firstChild) { chatHistory.firstChild.remove(); } return; } let context = document.getElementById('chat-history').context; if (context === undefined) { context = ""; } // Create user message element and append to chat history let chatHistory = document.getElementById('chat-history'); let userMessageDiv = document.createElement('div'); userMessageDiv.className = 'mb-2 user-message'; userMessageDiv.innerText = input; chatHistory.appendChild(userMessageDiv); copyTextToClipboard(userMessageDiv); // Create response container let responseDiv = document.createElement('div'); responseDiv.className = 'response-message mb-2 text-start'; responseDiv.style.minHeight = '3em'; // make sure div does not shrink if we cancel the request when no text has been generated yet let spinner = document.createElement('div'); spinner.className = 'spinner-border text-light'; spinner.setAttribute('role', 'status'); responseDiv.appendChild(spinner); chatHistory.appendChild(responseDiv); // create button to stop text generation sendButton.innerHTML = "Stop"; interrupt = new AbortController(); // change autoScroller to keep track of our new responseDiv autoScroller.observe(responseDiv); Query(context + " " + input, (word) => { // add word to response responseDiv.innerHTML = DOMPurify.sanitize(marked.parse(word)); // Append word to response container }).then(() => { chatHistory.context = responseDiv.innerHTML; copyTextToClipboard(responseDiv, true); sendButton.innerHTML = "Send"; spinner.remove(); interrupt = undefined; }).catch(error => { if (error !== 'Stop button pressed') { console.error(error); } sendButton.innerHTML = "Send"; spinner.remove(); interrupt = undefined; }); // Clear user input document.getElementById('user-input').value = ''; } const preCannedQueries = { "1": "Tell me about the lighthouse of Alexandria", "2": "Did the lighthouse of Alexandria existed at the same time the library of Alexandria existed?", "3": "How did the Pharos lighthouse impact ancient maritime trade?", "4": "Tell me about Constantinople?", }; // Event listener for Ctrl + Enter or CMD + Enter document.getElementById('user-input').addEventListener('keydown', function (e) { if (e.ctrlKey) { if (e.key === 'Enter') { submitRequest(); } else { const query = preCannedQueries[e.key]; if (query) { document.getElementById('user-input').value = query; submitRequest(); } } } }); function cleanup_text(text) { const assistantText = text.slice(text.indexOf('assistant|>') + 11); return assistantText; } function getConfig() { const query = window.location.search.substring(1); var config = { model: "schmuell/TinyLlama-1.1B-Chat-v1.0-int4", provider: "webgpu", isPhi2: false, needsExternalData: false, layers: 22, } let vars = query.split("&"); for (var i = 0; i < vars.length; i++) { let pair = vars[i].split("="); if (pair[0] in config) { config[pair[0]] = decodeURIComponent(pair[1]); } else if (pair[0].length > 0) { throw new Error("unknown argument: " + pair[0]); } } if (config.model.includes("phi2")) { config.isPhi2 = true; config.layers = 32; } config.needsExternalData = config.model.includes("-fp16"); return config; } const config = getConfig(); let pipe; async function Query(query, cb) { // Define the list of messages const messages = [ { "role": "system", "content": "You are a friendly assistant." }, // { "role": "user", "content": "Tell me about the lighthouse of Alexandria" }, { "role": "user", "content": query }, ] // Construct the prompt let prompt; if (config.isPhi2) { prompt = query; } else { prompt = pipe.tokenizer.apply_chat_template(messages, { tokenize: false, add_generation_prompt: true, }); } // Generate a response const start = performance.now(); const result = await pipe(prompt, { max_new_tokens: 256, temperature: 0.7, do_sample: true, top_k: 15, callback_function: function (beams) { const decodedText = pipe.tokenizer.decode(beams[0].output_token_ids, { skip_special_tokens: true, }); cb(cleanup_text(decodedText)); } }); const stop = performance.now(); console.log(`took ${((stop - start) / 1000).toFixed(1)}sec`); } async function LoadModel() { env.backends.onnx.wasm.numThreads = 1; env.allowRemoteModels = true; env.backends.onnx.wasm.wasmPaths = 'transformers/'; const model = config.model; let options; if (config.isPhi2) { // slighly different setup for phi2 options = { quantized: false, session_options: { executionProviders: [config.provider], preferredOutputLocation: {}, } } } else { options = { quantized: config.provider == "wasm" ? true : false, session_options: { executionProviders: [config.provider], preferredOutputLocation: {}, } } if (config.provider == "webgpu") { for (let i = 0; i < config.layers; ++i) { options.session_options.preferredOutputLocation[`present.${i}.key`] = 'gpu-buffer'; options.session_options.preferredOutputLocation[`present.${i}.value`] = 'gpu-buffer'; } } } if (config.needsExternalData) { options.session_options.externalData = [ { data: 'onnx/decoder_model_merged.onnx.data', path: 'decoder_model_merged.onnx.data' }, ]; } const start = performance.now(); log("Loading model ... "); pipe = await pipeline('text-generation', model, options); log(`done, ${((performance.now() - start) / 1000).toFixed(1)}sec`); } async function hasFp16() { try { const adapter = await navigator.gpu.requestAdapter() return adapter.features.has('shader-f16') } catch (e) { return false } } window.onload = () => { hasFp16().then((fp16) => { if (fp16) { LoadModel().then(() => { adjustPadding(); sendButton.addEventListener('click', submitRequest); const userInput = document.getElementById('user-input'); document.getElementById("status").style.display = "none"; userInput.focus(); }); } else { log("Your GPU or Browser doesn't support webgpu/f16"); } }); }