Spaces:

webai-community
/

demos

Running

App Files Files Community

Yang Gu commited on Apr 29, 2024

Commit

fa2d4e3

1 Parent(s): 37cf2c3

Support ort-phi3 and polish ort-phi2

Browse files

Files changed (7) hide show

demo/ort-phi2/index.html +3 -3
demo/ort-phi2/models/phi2-int4/tokenizer.json +0 -0
demo/ort-phi2/models/phi2-int4/tokenizer_config.json +323 -0
demo/ort-phi3/index.html +375 -0
demo/ort-phi3/models/phi3-int4/tokenizer.json +0 -0
demo/ort-phi3/models/phi3-int4/tokenizer_config.json +349 -0
main.js +12 -9

demo/ort-phi2/index.html CHANGED Viewed

@@ -7,8 +7,8 @@
 <body>
   <script src="../../util.js"></script>
-  <script src="https://cdn.jsdelivr.net/npm/onnxruntime-web@dev/dist/ort.webgpu.min.js"></script>
-  <!-- <script src="https://wp-27.sh.intel.com/workspace/project/onnxruntime/js/web/dist/ort.webgpu.min.js"> </script> -->
   <script type="module">
     import { AutoTokenizer, env } from '../../transformers/transformers.js';
@@ -88,7 +88,7 @@
         const model_config = JSON.parse(textDecoder.decode(json_bytes));
         const model_bytes = await fetchAndCache(modelPath + "/phi2-int4.onnx");
-        const externaldata = (model.externaldata) ? await fetchAndCache(modelPath + '/phi2-int4.onnx.data') : false;
         let modelSize = model_bytes.byteLength;
         if (externaldata) {
           modelSize += externaldata.byteLength;

 <body>
   <script src="../../util.js"></script>
+  <!-- <script src="https://cdn.jsdelivr.net/npm/onnxruntime-web@dev/dist/ort.webgpu.min.js"></script> -->
+  <script src="https://wp-27.sh.intel.com/workspace/project/onnxruntime/js/web/dist/ort.webgpu.min.js"></script>
   <script type="module">
     import { AutoTokenizer, env } from '../../transformers/transformers.js';
         const model_config = JSON.parse(textDecoder.decode(json_bytes));
         const model_bytes = await fetchAndCache(modelPath + "/phi2-int4.onnx");
+        const externaldata = (model.externaldata) ? await fetchAndCache(modelPath + '/phi2-int4.data') : false;
         let modelSize = model_bytes.byteLength;
         if (externaldata) {
           modelSize += externaldata.byteLength;

demo/ort-phi2/models/phi2-int4/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

demo/ort-phi2/models/phi2-int4/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,323 @@

+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50257": {
+      "content": "                               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "                         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50280": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50281": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50282": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50283": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50284": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50285": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50286": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50287": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50288": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50289": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50290": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50291": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50292": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50293": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50294": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 2048,
+  "tokenizer_class": "CodeGenTokenizer",
+  "unk_token": "<|endoftext|>"
+}

demo/ort-phi3/index.html ADDED Viewed

	@@ -0,0 +1,375 @@

+<!DOCTYPE html>
+<html>
+<head>
+  <title>Example</title>
+</head>
+<body>
+  <script src="../../util.js"></script>
+  <!-- <script src="https://cdn.jsdelivr.net/npm/onnxruntime-web@dev/dist/ort.webgpu.min.js"></script> -->
+  <script src="https://wp-27.sh.intel.com/workspace/project/onnxruntime/js/web/dist/ort.webgpu.min.js"></script>
+  <script type="module">
+    import { AutoTokenizer, env } from '../../transformers/transformers.js';
+    function log(i) { console.log(i); document.getElementById('status').innerText += `\n${i}`; }
+    const MODELS = {
+      "phi3": { name: "phi3", path: "phi3-int4", externaldata: true },
+    }
+    function getConfig() {
+      const query = window.location.search.substring(1);
+      var config = {
+        model: "phi3",
+        provider: "webgpu",
+        profiler: 0,
+        verbose: 0,
+        threads: 1,
+        trace: 0,
+        csv: 0,
+        max_tokens: 256,
+        local: 1,
+      }
+      let vars = query.split("&");
+      for (var i = 0; i < vars.length; i++) {
+        let pair = vars[i].split("=");
+        if (pair[0] in config) {
+          const key = pair[0];
+          const value = decodeURIComponent(pair[1]);
+          if (typeof config[key] == "number") {
+            config[key] = parseInt(value);
+          }
+          else {
+            config[key] = value;
+          }
+        } else if (pair[0].length > 0) {
+          throw new Error("unknown argument: " + pair[0]);
+        }
+      }
+      if (MODELS[config.model] !== undefined) {
+        config.model = MODELS[config.model];
+      }
+      return config;
+    }
+    class LLM {
+      sess = undefined;
+      profiler = false;
+      trace = false;
+      feed = {};
+      output_tokens = [];
+      eos = 2;
+      need_position_ids = true;
+      stop = false;
+      kv_dims = [];
+      dtype = "float16";
+      constructor() {
+      }
+      async load(model, options) {
+        const provider = options.provider || "webgpu";
+        const verbose = options.verbose;
+        const local = options.local;
+        this.profiler = options.profiler;
+        this.trace = options.trace;
+        const modelPath = getModelsPath() + model.path;
+        log(`loading... ${model.name},  ${provider}`);
+        const json_bytes = await fetchAndCache(modelPath + "/config.json");
+        let textDecoder = new TextDecoder();
+        const model_config = JSON.parse(textDecoder.decode(json_bytes));
+        const model_bytes = await fetchAndCache(modelPath + "/phi3-int4.onnx");
+        const externaldata = (model.externaldata) ? await fetchAndCache(modelPath + '/phi3-int4.data') : false;
+        let modelSize = model_bytes.byteLength;
+        if (externaldata) {
+          modelSize += externaldata.byteLength;
+        }
+        log(`model size ${Math.round(modelSize / 1024 / 1024)} MB`);
+        const opt = {
+          executionProviders: [provider],
+          preferredOutputLocation: {},
+        };
+        switch (provider) {
+          case "webgpu":
+            if (!("gpu" in navigator)) {
+              throw new Error("webgpu is NOT supported");
+            }
+            for (let i = 0; i < model_config.num_hidden_layers; ++i) {
+              opt.preferredOutputLocation[`present.${i}.key`] = 'gpu-buffer';
+              opt.preferredOutputLocation[`present.${i}.value`] = 'gpu-buffer';
+            }
+            break;
+          case "webnn":
+            if (!("ml" in navigator)) {
+              throw new Error("webnn is NOT supported");
+            }
+            break;
+        }
+        if (externaldata !== undefined) {
+          opt.externalData = [
+            {
+              data: externaldata,
+              path: 'decoder_model_merged.onnx.data'
+            },
+          ]
+        }
+        if (verbose) {
+          opt.logSeverityLevel = 0;
+          opt.logVerbosityLevel = 0;
+          ort.env.logLevel = "verbose";
+          ort.env.debug = true;
+        }
+        ort.env.webgpu.profiling = {};
+        if (this.profiler) {
+          opt.enableProfiling = true;
+          ort.env.webgpu.profilingMode = 'default';
+          ort.env.webgpu.profiling.mode = 'default';
+        }
+        this.sess = await ort.InferenceSession.create(model_bytes, opt);
+        if (this.trace) {
+          ort.env.trace = true;
+          ort.env.webgpu.profiling.ondata = (version, inputsMetadata, outputsMetadata, kernelId, kernelType,
+            kernelName, programName, startTime, endTime) => { };
+        }
+        this.eos = model_config.eos_token_id;
+        this.kv_dims = [1, model_config.num_key_value_heads, 0, model_config.hidden_size / model_config.num_attention_heads];
+        this.dtype = config.model.dtype || "float16";
+        this.num_layers = model_config.num_hidden_layers;
+        this.initilize_feed();
+      }
+      initilize_feed() {
+        this.feed = {};
+        const empty = (this.dtype === "float16") ? new Uint16Array() : [];
+        for (let i = 0; i < this.num_layers; ++i) {
+          this.feed[`past_key_values.${i}.key`] = new ort.Tensor(this.dtype, empty, this.kv_dims)
+          this.feed[`past_key_values.${i}.value`] = new ort.Tensor(this.dtype, empty, this.kv_dims)
+        }
+        this.output_tokens = [];
+      }
+      argmax(t) {
+        const arr = t.data;
+        const start = t.dims[2] * (t.dims[1] - 1);
+        let max = arr[start];
+        let maxidx = 0;
+        for (let i = 0; i < t.dims[2]; i++) {
+          const val = arr[i + start];
+          if (!isFinite(val)) {
+            throw new Error("found infinitive in logits");
+          }
+          if (val > max) {
+            max = arr[i + start];
+            maxidx = i;
+          }
+        }
+        return maxidx;
+      }
+      update_kv_cache(feed, outputs) {
+        for (const name in outputs) {
+          if (name.startsWith('present')) {
+            let newName = name.replace('present', 'past_key_values');
+            // free old gpu buffer
+            const t = feed[newName];
+            if (t.location === 'gpu-buffer') {
+              t.dispose();
+            }
+            feed[newName] = outputs[name];
+          }
+        }
+      }
+      abort() {
+        this.stop = true;
+      }
+      async generate(tokens, callback, options) {
+        const keep_cache = options.keep_cache;
+        const max_tokens = options.max_tokens || 256;
+        const feed = this.feed;
+        const input_ids = new ort.Tensor('int64', BigInt64Array.from(tokens.map(BigInt)), [1, tokens.length]);
+        feed['input_ids'] = input_ids;
+        this.stop = false;
+        if (keep_cache) {
+          this.output_tokens.push(...input_ids)
+        } else {
+          this.initilize_feed();
+          this.output_tokens = Array.from(feed['input_ids'].data);
+        }
+        let last_token = 0n;
+        let seqlen = this.output_tokens.length;
+        if (this.need_position_ids) {
+          if (keep_cache) {
+            feed['position_ids'] = new ort.Tensor('int64', BigInt64Array.from({ length: seqlen }, (_, i) => BigInt(i)), [1, input_ids.length]);
+          } else {
+            feed['position_ids'] = new ort.Tensor('int64', BigInt64Array.from({ length: seqlen }, (_, i) => BigInt(i)), [1, seqlen]);
+          }
+        }
+        while (last_token != this.eos && seqlen < max_tokens && !this.stop) {
+          seqlen = this.output_tokens.length;
+          feed['attention_mask'] = new ort.Tensor('int64', BigInt64Array.from({ length: seqlen }, () => 1n), [1, seqlen]);
+          let outputs;
+          if (this.trace) {
+            console.timeStamp("RUN-BEGIN");
+            outputs = await this.sess.run(feed);
+            console.timeStamp("RUN-END");
+          } else {
+            outputs = await this.sess.run(feed);
+          }
+          last_token = BigInt(this.argmax(outputs.logits));
+          this.output_tokens.push(last_token);
+          if (callback && !this.profiler) {
+            callback(this.output_tokens);
+          }
+          this.update_kv_cache(feed, outputs);
+          feed['input_ids'] = new ort.Tensor('int64', BigInt64Array.from([last_token]), [1, 1]);
+          if (this.need_position_ids) {
+            feed['position_ids'] = new ort.Tensor('int64', BigInt64Array.from([BigInt(seqlen)]), [1, 1]);
+          }
+        }
+        if (this.profiler) {
+          this.sess.endProfiling();
+        }
+        return this.output_tokens;
+      }
+    }
+    const config = getConfig();
+    env.localModelPath = 'models';
+    env.allowRemoteModels = config.local == 0;
+    env.allowLocalModels = config.local == 1;
+    ort.env.wasm.numThreads = config.threads;
+    ort.env.wasm.simd = true;
+    const cons_log = [];
+    if (config.profiler === 2) {
+      console.log = function (message) {
+        if (!message.includes('_fence_')) {
+          cons_log.push(message);
+        }
+      };
+    }
+    const tokenizer = await AutoTokenizer.from_pretrained(config.model.path);
+    function create_download_link(cons_log) {
+      if (cons_log.length > 0) {
+        let link = document.getElementById('download').childNodes[0];
+        if (link === undefined) {
+          link = document.createElement("a", "download-link");
+          link.download = "profiler.log";
+          link.innerText = "Download";
+          document.getElementById('download').appendChild(link);
+        }
+        const base64 = btoa(cons_log.join('\n'));
+        link.href = `data:application/json;base64,${base64}`;
+      }
+    }
+    async function fetchAndCache(url) {
+      try {
+        const cache = await caches.open("onnx");
+        let cachedResponse = await cache.match(url);
+        if (cachedResponse == undefined) {
+          await cache.add(url);
+          cachedResponse = await cache.match(url);
+          log(`${url} (network)`);
+        } else {
+          log(`${url} (cached)`);
+        }
+        const data = await cachedResponse.arrayBuffer();
+        return data;
+      } catch (error) {
+        log(`${url} (network)`);
+        return await fetch(url).then(response => response.arrayBuffer());
+      }
+    }
+    function token_to_text(tokenizer, tokens, startidx) {
+      const txt = tokenizer.decode(tokens.slice(startidx), { skip_special_tokens: true, });
+      return txt;
+    }
+    const llm = new LLM();
+    async function main() {
+      const model = config.model;
+      await llm.load(model, {
+        provider: config.provider,
+        verbose: config.verbose,
+        profiler: config.profiler,
+        trace: config.trace,
+        local: config.local,
+      });
+      document.getElementById('status').innerText = "";
+      const query = "Tell me about Constantinople.";
+      let prompt;
+      if (model.name.includes('phi3')) {
+        prompt = `User:${query}\nAssistant:`;
+      } else {
+        prompt = `"<|system|>\nYou are a friendly assistant.</s>\n<|user|>\n${query}</s>\n<|assistant|>\n`;
+      }
+      const { input_ids } = await tokenizer(prompt, { return_tensor: false, padding: true, truncation: true });
+      const start_timer = performance.now();
+      const output_tokens = await llm.generate(input_ids, (output_tokens) => {
+        document.getElementById('result').innerText = token_to_text(tokenizer, output_tokens, input_ids.length);
+      }, {});
+      const took = (performance.now() - start_timer) / 1000;
+      const txt = token_to_text(tokenizer, output_tokens, input_ids.length);
+      const seqlen = output_tokens.length;
+      document.getElementById('result').innerText = txt;
+      const perf = `${seqlen} tokens in ${took.toFixed(1)}sec, ${(seqlen / took).toFixed(2)} tokens/sec`;
+      console.log(perf + " @@1");
+      document.getElementById('perf').innerText = perf;
+      if (config.csv) {
+        log(`${model.name},${took.toFixed(2)},${(seqlen / took).toFixed(3)},${seqlen},@@2`);
+      }
+    }
+    try {
+      await main();
+    } catch (error) {
+      console.error(error);
+      document.getElementById('result').innerText = error.message;
+    } finally {
+      create_download_link(cons_log);
+    }
+  </script>
+  <div id="status"></div>
+  <br />
+  <div id="result"></div>
+  <br />
+  <div id="perf"></div>
+  <br />
+  <div id="download"></div>
+  <br />
+</body>
+</html>

demo/ort-phi3/models/phi3-int4/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

demo/ort-phi3/models/phi3-int4/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,349 @@

+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|step|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|function_output|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|function_call|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|raw|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|continue|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32011": {
+      "content": "<|function_list|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32012": {
+      "content": "<|calc|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32013": {
+      "content": "<|code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32014": {
+      "content": "<|/code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32015": {
+      "content": "<|summary|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32016": {
+      "content": "<|resource|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32017": {
+      "content": "<|assistant_mask|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32018": {
+      "content": "<|start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32019": {
+      "content": "<|message|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32020": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32021": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32022": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32023": {
+      "content": "<|meta_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32024": {
+      "content": "<|ipynb_marker|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32025": {
+      "content": "<|diff_marker|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32026": {
+      "content": "<|ghissue|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32027": {
+      "content": "<|ghreview|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32028": {
+      "content": "<|disc_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32029": {
+      "content": "<|disc_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32030": {
+      "content": "<|disc_thread|><|query|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32031": {
+      "content": "<|/query|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32032": {
+      "content": "<|data|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32033": {
+      "content": "<|/data|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32034": {
+      "content": "<|sys|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32035": {
+      "content": "<|/sys|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32036": {
+      "content": "<|inst|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32037": {
+      "content": "<|/inst|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|/inst|>"
+  ],
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "legacy": false,
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

main.js CHANGED Viewed

@@ -49,11 +49,9 @@ function createElem(tag, attrs = {}, children = []) {
   return elem;
 }
-// todo: Musicgen
-const pageCategories = [
   {
-    title: `ONNX Runtime`,
     description: `ONNX Runtime`,
     demos: {
       'ort-phi2': {
@@ -61,6 +59,11 @@ const pageCategories = [
         description: `phi2 from Microsoft`,
         filename: "ort-phi2",
       },
       'ort-sam': {
         name: 'Segment Anything',
         description: `Segment Anything from https://github.com/guschmue/ort-webgpu/tree/master/segment-anything`,
@@ -84,7 +87,7 @@ const pageCategories = [
     },
   },
   {
-    title: `TFLite`,
     description: `TFLite`,
     demos: {
       'tflite-gemma': {
@@ -95,7 +98,7 @@ const pageCategories = [
     },
   },
   {
-    title: 'Transformers.js',
     description: 'Transformers.js',
     demos: {
       benchmark: {
@@ -131,7 +134,7 @@ const pageCategories = [
     },
   },
   {
-    title: 'TVM',
     description: 'TVM',
     demos: {
       sd: {
@@ -32475,7 +32478,7 @@ function setSampleIFrameURL(e, demoInfo) {
 // from those keys to each demo.
 const samplesByKey = new Map();
 // Generate the list of demos
-for (const { title, description, demos } of pageCategories) {
   for (const [key, demoInfo] of Object.entries(demos)) {
     samplesByKey.set(key, demoInfo);
   }
@@ -32484,7 +32487,7 @@ for (const { title, description, demos } of pageCategories) {
       createElem('div', { className: 'sampleCategory' }, [
         createElem('h3', {
           style: { 'margin-top': '5px' },
-          textContent: title,
           dataset: { tooltip: description },
         }),
       ]),

   return elem;
 }
+const demoCategories = [
   {
+    name: `ONNX Runtime`,
     description: `ONNX Runtime`,
     demos: {
       'ort-phi2': {
         description: `phi2 from Microsoft`,
         filename: "ort-phi2",
       },
+      'ort-phi3': {
+        name: 'phi3',
+        description: `phi3 from Microsoft`,
+        filename: "ort-phi3",
+      },
       'ort-sam': {
         name: 'Segment Anything',
         description: `Segment Anything from https://github.com/guschmue/ort-webgpu/tree/master/segment-anything`,
     },
   },
   {
+    name: `TFLite`,
     description: `TFLite`,
     demos: {
       'tflite-gemma': {
     },
   },
   {
+    name: 'Transformers.js',
     description: 'Transformers.js',
     demos: {
       benchmark: {
     },
   },
   {
+    name: 'TVM',
     description: 'TVM',
     demos: {
       sd: {
 // from those keys to each demo.
 const samplesByKey = new Map();
 // Generate the list of demos
+for (const { name, description, demos } of demoCategories) {
   for (const [key, demoInfo] of Object.entries(demos)) {
     samplesByKey.set(key, demoInfo);
   }
       createElem('div', { className: 'sampleCategory' }, [
         createElem('h3', {
           style: { 'margin-top': '5px' },
+          textContent: name,
           dataset: { tooltip: description },
         }),
       ]),