Upload 5 files

app.js

@@ -0,0 +1,105 @@
+const go = new Go();
+
+document.addEventListener("DOMContentLoaded", function () {
+    const tokenizeButton = document.getElementById("js-tokenize");
+    const loadingMessage = document.getElementById("js-message");
+
+    WebAssembly.instantiateStreaming(fetch("main.wasm"), go.importObject).then(result => {
+        go.run(result.instance);
+
+        tokenizeButton.disabled = false;
+        loadingMessage.style.display = "none";
+    });
+
+    const pinchZoom = document.getElementById("js-mermaid");
+
+    pinchZoom.setTransform({
+        scale: 1,
+        x: 0,
+        y: 220,
+        allowChangeEvent: false,
+    });
+
+    tokenizeButton.addEventListener("click", () => tokenizeText());
+});
+
+let data = [];
+
+async function tokenizeText() {
+    const input = document.getElementById("js-input").value.trim();
+    const model = document.getElementById("js-model").value;
+    const vocab = parseInt(document.getElementById("js-vocab").value, 10) || -1;
+
+    const message = document.getElementById("js-message");
+    const mermaid = document.getElementById("js-mermaid");
+
+    if (!input || typeof tokenizeWeb == "undefined") {
+        return;
+    }
+
+    const result = JSON.parse(tokenizeWeb(input, model, vocab));
+
+    if (!Array.isArray(result) || result.length < 1) {
+        message.textContent = "No tokens found.";
+        mermaid.innerHTML = "";
+
+        return;
+    }
+
+    data = result;
+
+    updateTabs(result);
+    showResult(0);
+}
+
+function updateTabs() {
+    const tabs = document.getElementById("js-tabs");
+
+    tabs.innerHTML = "";
+
+    data.forEach((chunk, index) => {
+        const tab = document.createElement("button");
+
+        tab.classList.add("mbpe-tab");
+        tab.dataset.index = index.toString();
+        tab.onclick = () => {
+            tab.scrollIntoView({
+                behavior: 'smooth',
+                block: 'nearest',
+                inline: 'center',
+            });
+
+            showResult(index);
+        }
+
+        const list = document.createElement("ol");
+
+        list.classList.add("mbpe-chunk");
+
+        chunk.Segmentations[chunk.Segmentations.length - 1].map(token => {
+            const item = document.createElement("li");
+
+            item.textContent = token.Token;
+            item.classList.add("mbpe-token");
+            item.title = token.ID;
+
+            list.appendChild(item)
+        });
+
+        tab.appendChild(list);
+        tabs.appendChild(tab);
+    });
+}
+
+function showResult(index) {
+    if (index < 0 || index >= data.length) {
+        return;
+    }
+
+    mermaid.render("mermaidDiagram", data[index].Mermaid).then(({ svg }) => {
+        document.getElementById("js-mermaid").innerHTML = svg;
+    });
+
+    document.querySelectorAll(".mbpe-tab").forEach(tab => tab.classList.remove("mbpe-tab--active"));
+    document.querySelector(`.mbpe-tab[data-index="${index}"]`).classList.add("mbpe-tab--active");
+}

index.html

@@ -5,221 +5,44 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>mbpe-dyn</title>
     <script src="https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/pinch-zoom-min.js"></script>
     <script src="wasm_exec.js"></script>
-    <script>
-        const go = new Go();
-
-        document.addEventListener("DOMContentLoaded", function () {
-            const tokenizeButton = document.querySelector("button[onclick='tokenizeText()']");
-            const loadingMessage = document.getElementById("loadingMessage");
-
-            WebAssembly.instantiateStreaming(fetch("main.wasm"), go.importObject).then(result => {
-                go.run(result.instance);
-
-                tokenizeButton.disabled = false;
-                loadingMessage.style.display = "none";
-            });
-        });
-
-        async function tokenizeText() {
-            const input = document.getElementById("textInput").value.trim();
-            const modelChoice = document.getElementById("modelSelect").value;
-            const vocabSize = parseInt(document.getElementById("vocabSize").value, 10) || -1;
-
-            if (!input) {
-                alert("Please enter text to tokenize.");
-                return;
-            }
-
-            if (typeof tokenizeWeb !== "undefined") {
-                const resultJSON = tokenizeWeb(input, modelChoice, vocabSize);
-
-                try {
-                    const result = JSON.parse(resultJSON);
-                    console.log(result);
-
-                    if (Array.isArray(result) && result.length > 0) {
-                        displayTabs(result);
-                        showResult(0); // Show first tab by default
-                    } else {
-                        document.getElementById("code").innerText = "No tokens found.";
-                        document.getElementById("mermaidContainer").innerHTML = "";
-                    }
-                } catch (e) {
-                    console.error("Error parsing result:", e);
-                }
-            }
-        }
-
-        function displayTabs(results) {
-            const tabsContainer = document.getElementById("wordTabs");
-            tabsContainer.innerHTML = "";
-
-            results.forEach((entry, index) => {
-                const finalSegmentation = entry.Segmentations[entry.Segmentations.length - 1];
-                const label = finalSegmentation.map(t => t.Token).join("");
-
-                const tabButton = document.createElement("button");
-                tabButton.textContent = label;
-                tabButton.classList.add("word-tab");
-                tabButton.dataset.index = index;
-                tabButton.onclick = () => showResult(index);
-
-                tabsContainer.appendChild(tabButton);
-            });
-        }
-
-        function showResult(index) {
-            const results = JSON.parse(tokenizeWeb(
-                document.getElementById("textInput").value.trim(),
-                document.getElementById("modelSelect").value,
-                parseInt(document.getElementById("vocabSize").value, 10) || -1
-            ));
-
-            const selectedResult = results[index];
-            const diagramCode = selectedResult.Mermaid.trim();
-
-            if (diagramCode) {
-                mermaid.render("mermaidDiagram", diagramCode).then(({ svg }) => {
-                    document.getElementById("mermaidContainer").innerHTML = svg;
-                }).catch(error => {
-                    console.error("Mermaid rendering error:", error);
-                });
-            }
-
-            // Update active tab
-            document.querySelectorAll(".word-tab").forEach(tab => tab.classList.remove("active"));
-            document.querySelector(`.word-tab[data-index="${index}"]`).classList.add("active");
-        }
-    </script>
-    <style>
-        /* General Styles */
-        body {
-            font-family: Arial, sans-serif;
-            background: #f8f9fa;
-            color: #333;
-            text-align: center;
-            padding: 20px;
-        }
-
-        h1 {
-            color: #007bff;
-        }
-
-        /* Input and Select */
-        .input-group {
-            display: flex;
-            justify-content: center;
-            gap: 10px;
-            margin-bottom: 20px;
-        }
-
-        input, select, button {
-            padding: 10px;
-            border: 1px solid #ccc;
-            border-radius: 5px;
-            font-size: 16px;
-        }
-
-        input {
-            width: 300px;
-        }
-
-        select {
-            width: 150px;
-            background: white;
-            cursor: pointer;
-        }
-
-        button {
-            background: #007bff;
-            color: white;
-            font-weight: bold;
-            cursor: pointer;
-            transition: 0.3s;
-            border: none;
-        }
-
-        button:hover {
-            background: #0056b3;
-        }
-
-        button:disabled {
-            background: #ccc;
-            color: #666;
-            cursor: not-allowed;
-        }
-
-        #modelSelect {
-            width: 80px;
-        }
-
-        #vocabSize {
-            width: 110px;
-        }
-
-        #loadingMessage {
-            text-align: center;
-            font-weight: bold;
-            color: #555;
-            margin-top: 10px;
-        }
-
-        /* Word Tabs */
-        .word-tabs {
-            display: flex;
-            justify-content: center;
-            gap: 10px;
-            margin-top: 20px;
-            margin-bottom: 20px;
-            flex-wrap: wrap;
-        }
-
-        .word-tab {
-            padding: 8px 15px;
-            background: #e1ecf4;
-            border: none;
-            border-radius: 20px;
-            cursor: pointer;
-            font-weight: bold;
-            transition: 0.3s;
-        }
-
-        .word-tab.active {
-            background: #007bff;
-            color: white;
-        }
-    </style>
+    <script src="app.js"></script>
+    <link rel="stylesheet" href="style.css">
 </head>
 <body>
-
-<h1>mbpe-dyn</h1>
-
-<div class="input-group">
-    <select id="modelSelect">
-        <option value="m000">m000</option>
-        <option value="m010">m010</option>
-        <option value="m020">m020</option>
-        <option value="m030">m030</option>
-        <option value="m040">m040</option>
-        <option value="m050">m050</option>
-        <option value="m060">m060</option>
-        <option value="m070">m070</option>
-        <option value="m080">m080</option>
-        <option value="m090">m090</option>
-        <option value="m100" selected>m100</option>
-    </select>
-    <select id="vocabSize">
-        <option value="16384">2¹⁴ (16K)</option>
-        <option value="32768">2¹⁵ (32K)</option>
-        <option value="65536">2¹⁶ (64K)</option>
-        <option value="131072" selected>2¹⁷ (128K)</option>
-    </select>
-    <input type="text" id="textInput" placeholder="Enter text here" value="airsickness">
-    <button onclick="tokenizeText()" disabled>Tokenize</button>
+<div class="mbpe-panel">
+    <div class="mbpe-panel__form">
+        <input type="text" id="js-input" class="mbpe-input" placeholder="Enter text here" value="I dislike flying because of my airsickness"v>
+        <div class="mbpe-panel__row">
+            <div class="mbpe-panel__group">
+                <select id="js-model" class="mbpe-select">
+                    <option value="m000">m000</option>
+                    <option value="m010">m010</option>
+                    <option value="m020">m020</option>
+                    <option value="m030">m030</option>
+                    <option value="m040">m040</option>
+                    <option value="m050">m050</option>
+                    <option value="m060">m060</option>
+                    <option value="m070">m070</option>
+                    <option value="m080">m080</option>
+                    <option value="m090">m090</option>
+                    <option value="m100" selected>m100</option>
+                </select>
+                <select id="js-vocab" class="mbpe-select">
+                    <option value="16384">2¹⁴ (16K)</option>
+                    <option value="32768">2¹⁵ (32K)</option>
+                    <option value="65536">2¹⁶ (64K)</option>
+                    <option value="131072" selected>2¹⁷ (128K)</option>
+                </select>
+            </div>
+            <button id="js-tokenize" class="mbpe-button" disabled>Tokenize</button>
+        </div>
+    </div>
+    <div id="js-tabs" class="mbpe-panel__result">
+        <div id="js-message" class="mbpe-panel__message">Initializing tokenizers...</div>
+    </div>
 </div>
-<div id="loadingMessage">Initializing tokenizers...</div>
-<div id="wordTabs" class="word-tabs"></div>
-<div id="mermaidContainer"></div>
+<pinch-zoom id="js-mermaid" class="my-pinch-zoom mbpe-container"></pinch-zoom>
 </body>
 </html>

main.wasm

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8b0c692cf0fd88106ec4636e9fdfbc2ca6723efd104aeefb4c344780132a5c4
-size 43684316
+oid sha256:b2429e25bb515989962bf15bccd600abc2799582fbbce28edc50255a7b9080fe
+size 43689867

style.css

@@ -0,0 +1,176 @@
+:root {
+    --primary-color: #080808;
+    --secondary-color: #1d1d1d;
+    --background-color: #f8f9fa;
+    --border-color: #cccccc;
+}
+
+body {
+    box-sizing: border-box;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    width: 100%;
+    min-height: 100vh;
+    margin: 0;
+    font-family: Arial, sans-serif;
+    font-size: 16px;
+    text-align: center;
+    background: var(--background-color);
+    background:
+            linear-gradient(90deg, var(--dot-bg) calc(var(--dot-space) - var(--dot-size)), transparent 1%) center / var(--dot-space) var(--dot-space),
+            linear-gradient(var(--dot-bg) calc(var(--dot-space) - var(--dot-size)), transparent 1%) center / var(--dot-space) var(--dot-space),
+            var(--dot-color);
+    --dot-bg: white;
+    --dot-color: black;
+    --dot-size: 1px;
+    --dot-space: 22px;
+}
+
+.mbpe-panel {
+    position: absolute;
+    top: 0;
+    z-index: 100;
+    box-sizing: border-box;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    width: 800px;
+    max-width: calc(100vw - 40px);
+    margin: 20px;
+    overflow: hidden;
+    background-color: var(--background-color);
+    border: 1px solid var(--border-color);
+    border-radius: 5px;
+}
+
+.mbpe-panel__row {
+    display: flex;
+    flex-direction: row;
+    gap: 10px;
+    justify-content: space-between;
+    width: 100%;
+}
+
+.mbpe-panel__group {
+    display: flex; flex-direction: row; gap: 10px;
+}
+
+.mbpe-panel__form {
+    box-sizing: border-box;
+    display: flex;
+    flex-direction: column;
+    gap: 10px;
+    width: 100%;
+    padding: 10px;
+}
+
+.mbpe-input {
+    box-sizing: border-box;
+    padding: 10px;
+    font-size: 16px;
+    border: 1px solid var(--border-color);
+    border-radius: 5px;
+}
+
+.mbpe-select {
+    padding: 10px 35px 10px 10px;
+    font-size: 16px;
+    appearance: none;
+    cursor: pointer;
+    background-color: white;
+    background-image: url('data:image/svg+xml;utf8,<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M6 9L12 15L18 9" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" stroke="currentColor"/></svg>');
+    background-repeat: no-repeat;
+    background-position: right 10px center;
+    background-size: 18px;
+    border: 1px solid var(--border-color);
+    border-radius: 5px;
+}
+
+.mbpe-button {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    padding: 10px;
+    font-size: 0.875rem;
+    font-weight: 700;
+    color: white;
+    text-transform: uppercase;
+    letter-spacing: 0.2px;
+    cursor: pointer;
+    background-color: var(--primary-color);
+    border: none;
+    border-radius: 5px;
+    transition: 0.3s;
+}
+
+.mbpe-button:disabled {
+    cursor: not-allowed;
+}
+
+
+.mbpe-panel__message {
+    font-weight: bold;
+    color: var(--secondary-color);
+    text-align: center;
+}
+
+.mbpe-panel__result {
+    box-sizing: border-box;
+    display: flex;
+    flex-direction: row;
+    flex-wrap: nowrap;
+    gap: 4px;
+    align-items: center;
+    justify-content: start;
+    width: 100%;
+    max-width: calc(100vw - 40px);
+    min-height: 66px;
+    padding: 10px;
+    overflow: scroll;
+    border-top: 1px solid var(--border-color);
+}
+
+.mbpe-tab {
+    padding: 4px;
+    font-weight: bold;
+    cursor: pointer;
+    background: transparent;
+    border: 1px dashed var(--primary-color);
+    border-radius: 24px;
+    transition: 0.3s;
+}
+
+.mbpe-tab:hover {
+    background: var(--primary-color);
+}
+
+.mbpe-tab--active {
+    background: var(--primary-color);
+}
+
+.mbpe-chunk {
+    display: flex;
+    gap: 4px;
+    padding: 0;
+    margin: 0;
+    list-style: none;
+}
+
+.mbpe-token {
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+    justify-content: center;
+    padding: 8px;
+    font-size: 0.875rem;
+    background-color: var(--background-color);
+    border: 1px solid var(--primary-color);
+    border-radius: 20px;
+}
+
+.mbpe-container {
+    width: 100vw; height: 100vh;
+}