Spaces:

siro1
/

triton-for-llms

Sleeping

File size: 33,598 Bytes

cf7a8a2

<!DOCTYPE html>
<html lang="en"><head>
<script src="site_libs/clipboard/clipboard.min.js"></script>
<script src="site_libs/quarto-html/tabby.min.js"></script>
<script src="site_libs/quarto-html/popper.min.js"></script>
<script src="site_libs/quarto-html/tippy.umd.min.js"></script>
<link href="site_libs/quarto-html/tippy.css" rel="stylesheet">
<link href="site_libs/quarto-html/light-border.css" rel="stylesheet">
<link href="site_libs/quarto-html/quarto-syntax-highlighting-dark-8ea72dc5fed832574809a9c94082fbbb.css" rel="stylesheet" id="quarto-text-highlighting-styles"><meta charset="utf-8">
  <meta name="generator" content="quarto-1.6.40">

  <meta name="author" content="Matej Sirovatka">
  <meta name="dcterms.date" content="2025-02-21">
  <title>Open-Source AI Cookbook – Optimizing LLM Performance Using Triton</title>
  <meta name="apple-mobile-web-app-capable" content="yes">
  <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, minimal-ui">
  <link rel="stylesheet" href="site_libs/revealjs/dist/reset.css">
  <link rel="stylesheet" href="site_libs/revealjs/dist/reveal.css">
  <style>
    code{white-space: pre-wrap;}
    span.smallcaps{font-variant: small-caps;}
    div.columns{display: flex; gap: min(4vw, 1.5em);}
    div.column{flex: auto; overflow-x: auto;}
    div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
    ul.task-list{list-style: none;}
    ul.task-list li input[type="checkbox"] {
      width: 0.8em;
      margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */ 
      vertical-align: middle;
    }
    /* CSS for syntax highlighting */
    pre > code.sourceCode { white-space: pre; position: relative; }
    pre > code.sourceCode > span { line-height: 1.25; }
    pre > code.sourceCode > span:empty { height: 1.2em; }
    .sourceCode { overflow: visible; }
    code.sourceCode > span { color: inherit; text-decoration: inherit; }
    div.sourceCode { margin: 1em 0; }
    pre.sourceCode { margin: 0; }
    @media screen {
    div.sourceCode { overflow: auto; }
    }
    @media print {
    pre > code.sourceCode { white-space: pre-wrap; }
    pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
    }
    pre.numberSource code
      { counter-reset: source-line 0; }
    pre.numberSource code > span
      { position: relative; left: -4em; counter-increment: source-line; }
    pre.numberSource code > span > a:first-child::before
      { content: counter(source-line);
        position: relative; left: -1em; text-align: right; vertical-align: baseline;
        border: none; display: inline-block;
        -webkit-touch-callout: none; -webkit-user-select: none;
        -khtml-user-select: none; -moz-user-select: none;
        -ms-user-select: none; user-select: none;
        padding: 0 4px; width: 4em;
      }
    pre.numberSource { margin-left: 3em;  padding-left: 4px; }
    div.sourceCode
      { color: #f8f8f2;  }
    @media screen {
    pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
    }
    code span { color: #f8f8f2; } /* Normal */
    code span.al { color: #f07178; background-color: #2a0f15; font-weight: bold; } /* Alert */
    code span.an { color: #d4d0ab; } /* Annotation */
    code span.at { color: #00e0e0; } /* Attribute */
    code span.bn { color: #d4d0ab; } /* BaseN */
    code span.bu { color: #abe338; } /* BuiltIn */
    code span.cf { color: #ffa07a; font-weight: bold; } /* ControlFlow */
    code span.ch { color: #abe338; } /* Char */
    code span.cn { color: #ffd700; } /* Constant */
    code span.co { color: #f8f8f2; font-style: italic; } /* Comment */
    code span.cv { color: #ffd700; } /* CommentVar */
    code span.do { color: #f8f8f2; } /* Documentation */
    code span.dt { color: #ffa07a; } /* DataType */
    code span.dv { color: #d4d0ab; } /* DecVal */
    code span.er { color: #f07178; text-decoration: underline; } /* Error */
    code span.ex { color: #00e0e0; font-weight: bold; } /* Extension */
    code span.fl { color: #d4d0ab; } /* Float */
    code span.fu { color: #ffa07a; } /* Function */
    code span.im { color: #abe338; } /* Import */
    code span.in { color: #d4d0ab; } /* Information */
    code span.kw { color: #ffa07a; font-weight: bold; } /* Keyword */
    code span.op { color: #ffa07a; } /* Operator */
    code span.ot { color: #00e0e0; } /* Other */
    code span.pp { color: #dcc6e0; } /* Preprocessor */
    code span.re { color: #00e0e0; background-color: #f8f8f2; } /* RegionMarker */
    code span.sc { color: #abe338; } /* SpecialChar */
    code span.ss { color: #abe338; } /* SpecialString */
    code span.st { color: #abe338; } /* String */
    code span.va { color: #00e0e0; } /* Variable */
    code span.vs { color: #abe338; } /* VerbatimString */
    code span.wa { color: #dcc6e0; } /* Warning */
  </style>
  <link rel="stylesheet" href="site_libs/revealjs/dist/theme/quarto-5b48f34d633aed70c74c672477009ffc.css">
  <link href="site_libs/revealjs/plugin/quarto-line-highlight/line-highlight.css" rel="stylesheet">
  <link href="site_libs/revealjs/plugin/reveal-menu/menu.css" rel="stylesheet">
  <link href="site_libs/revealjs/plugin/reveal-menu/quarto-menu.css" rel="stylesheet">
  <link href="site_libs/revealjs/plugin/quarto-support/footer.css" rel="stylesheet">
  <style type="text/css">
    .reveal div.sourceCode {
      margin: 0;
      overflow: auto;
    }
    .reveal div.hanging-indent {
      margin-left: 1em;
      text-indent: -1em;
    }
    .reveal .slide:not(.center) {
      height: 100%;
    }
    .reveal .slide.scrollable {
      overflow-y: auto;
    }
    .reveal .footnotes {
      height: 100%;
      overflow-y: auto;
    }
    .reveal .slide .absolute {
      position: absolute;
      display: block;
    }
    .reveal .footnotes ol {
      counter-reset: ol;
      list-style-type: none; 
      margin-left: 0;
    }
    .reveal .footnotes ol li:before {
      counter-increment: ol;
      content: counter(ol) ". "; 
    }
    .reveal .footnotes ol li > p:first-child {
      display: inline-block;
    }
    .reveal .slide ul,
    .reveal .slide ol {
      margin-bottom: 0.5em;
    }
    .reveal .slide ul li,
    .reveal .slide ol li {
      margin-top: 0.4em;
      margin-bottom: 0.2em;
    }
    .reveal .slide ul[role="tablist"] li {
      margin-bottom: 0;
    }
    .reveal .slide ul li > *:first-child,
    .reveal .slide ol li > *:first-child {
      margin-block-start: 0;
    }
    .reveal .slide ul li > *:last-child,
    .reveal .slide ol li > *:last-child {
      margin-block-end: 0;
    }
    .reveal .slide .columns:nth-child(3) {
      margin-block-start: 0.8em;
    }
    .reveal blockquote {
      box-shadow: none;
    }
    .reveal .tippy-content>* {
      margin-top: 0.2em;
      margin-bottom: 0.7em;
    }
    .reveal .tippy-content>*:last-child {
      margin-bottom: 0.2em;
    }
    .reveal .slide > img.stretch.quarto-figure-center,
    .reveal .slide > img.r-stretch.quarto-figure-center {
      display: block;
      margin-left: auto;
      margin-right: auto; 
    }
    .reveal .slide > img.stretch.quarto-figure-left,
    .reveal .slide > img.r-stretch.quarto-figure-left  {
      display: block;
      margin-left: 0;
      margin-right: auto; 
    }
    .reveal .slide > img.stretch.quarto-figure-right,
    .reveal .slide > img.r-stretch.quarto-figure-right  {
      display: block;
      margin-left: auto;
      margin-right: 0; 
    }
  </style>
</head>
<body class="quarto-dark">
  <div class="reveal">
    <div class="slides">

<section id="title-slide" class="quarto-title-block center">
  <h1 class="title">Optimizing LLM Performance Using Triton</h1>

<div class="quarto-title-authors">
<div class="quarto-title-author">
<div class="quarto-title-author-name">
Matej Sirovatka 
</div>
</div>
</div>

  <p class="date">2025-02-21</p>
</section>
<section id="whoami" class="slide level2">
<h2><code>whoami</code></h2>
<ul>
<li>My name is Matej</li>
<li>I’m a Master’s student at the Brno University of Technology</li>
<li>I currently make GPUs go <code>brrrrrr</code> at Hugging Face 🤗</li>
</ul>
</section>
<section id="what-is-triton" class="slide level2">
<h2><code>What is Triton?</code></h2>
<ul>
<li>NVIDIA’s open-source programming language for GPU kernels</li>
<li>Designed for AI/ML workloads</li>
<li>Simplifies GPU programming compared to CUDA</li>
</ul>

<img data-src="media/optim_scale.png" class="center quarto-figure quarto-figure-center r-stretch"></section>
<section id="why-optimize-with-triton" class="slide level2">
<h2><code>Why Optimize with Triton?</code></h2>
<ul>
<li>Simple yet effective</li>
<li>Less headache than CUDA</li>
<li>GPUs go <code>brrrrrrr</code> 🚀</li>
<li>Feel cool when your kernel is faster than PyTorch 😎</li>
</ul>
</section>
<section id="example-problem-kl-divergence" class="slide level2">
<h2><code>Example Problem: KL Divergence</code></h2>
<ul>
<li>commonly used in LLMs for knowledge distillation</li>
<li>for probability distributions <span class="math inline">\(P\)</span> and <span class="math inline">\(Q\)</span>, the Kullback-Leibler divergence is defined as:</li>
</ul>
<p><span class="math display">\[
D_{KL}(P \| Q) = \sum_{i} P_i \log\left(\frac{P_i}{Q_i}\right)
\]</span></p>
<div class="sourceCode" id="cb1"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a></a><span class="im">import</span> torch</span>
<span id="cb1-2"><a></a><span class="im">from</span> torch.nn.functional <span class="im">import</span> kl_div</span>
<span id="cb1-3"><a></a></span>
<span id="cb1-4"><a></a><span class="kw">def</span> kl_div_torch(p: torch.Tensor, q: torch.Tensor) <span class="op">-&gt;</span> torch.Tensor:</span>
<span id="cb1-5"><a></a>    <span class="cf">return</span> kl_div(p, q, reduction<span class="op">=</span><span class="st">'none'</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
</section>
<section id="how-about-triton" class="slide level2">
<h2><code>How about Triton?</code></h2>
<div class="sourceCode" id="cb2"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a></a><span class="im">import</span> triton.language <span class="im">as</span> tl</span>
<span id="cb2-2"><a></a></span>
<span id="cb2-3"><a></a><span class="at">@triton.jit</span></span>
<span id="cb2-4"><a></a><span class="kw">def</span> kl_div_triton(</span>
<span id="cb2-5"><a></a>    p_ptr, q_ptr, output_ptr, n_elements, BLOCK_SIZE: tl.constexpr</span>
<span id="cb2-6"><a></a>):</span>
<span id="cb2-7"><a></a>    pid <span class="op">=</span> tl.program_id(<span class="dv">0</span>)</span>
<span id="cb2-8"><a></a>    block_start <span class="op">=</span> pid <span class="op">*</span> BLOCK_SIZE</span>
<span id="cb2-9"><a></a>    offsets <span class="op">=</span> block_start <span class="op">+</span> tl.arange(<span class="dv">0</span>, BLOCK_SIZE)</span>
<span id="cb2-10"><a></a>    mask <span class="op">=</span> offsets <span class="op">&lt;</span> n_elements</span>
<span id="cb2-11"><a></a>    </span>
<span id="cb2-12"><a></a>    p <span class="op">=</span> tl.load(p_ptr <span class="op">+</span> offsets, mask<span class="op">=</span>mask)</span>
<span id="cb2-13"><a></a>    q <span class="op">=</span> tl.load(q_ptr <span class="op">+</span> offsets, mask<span class="op">=</span>mask)</span>
<span id="cb2-14"><a></a>    </span>
<span id="cb2-15"><a></a>    output <span class="op">=</span> p <span class="op">*</span> (tl.log(p) <span class="op">-</span> tl.log(q))</span>
<span id="cb2-16"><a></a>    </span>
<span id="cb2-17"><a></a>    tl.store(output_ptr <span class="op">+</span> offsets, output, mask<span class="op">=</span>mask)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
</section>
<section id="how-to-integrate-with-pytorch" class="slide level2">
<h2><code>How to integrate with PyTorch?</code></h2>
<ul>
<li>Triton works with pointers</li>
<li>How to use our custom kernel with PyTorch autograd?</li>
</ul>
<div class="sourceCode" id="cb3"><pre class="sourceCode numberSource python number-lines code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a></a><span class="im">import</span> torch</span>
<span id="cb3-2"><a></a></span>
<span id="cb3-3"><a></a><span class="kw">class</span> VectorAdd(torch.autograd.Function):</span>
<span id="cb3-4"><a></a>    <span class="at">@staticmethod</span></span>
<span id="cb3-5"><a></a>    <span class="kw">def</span> forward(ctx, p, q):</span>
<span id="cb3-6"><a></a>        ctx.save_for_backward(q)</span>
<span id="cb3-7"><a></a>        output <span class="op">=</span> torch.empty_like(p)</span>
<span id="cb3-8"><a></a>        grid <span class="op">=</span> (<span class="bu">len</span>(p) <span class="op">+</span> <span class="dv">512</span> <span class="op">-</span> <span class="dv">1</span>) <span class="op">//</span> <span class="dv">512</span></span>
<span id="cb3-9"><a></a>        kl_div_triton[grid](p, q, output, <span class="bu">len</span>(p), BLOCK_SIZE<span class="op">=</span><span class="dv">512</span>)</span>
<span id="cb3-10"><a></a>        <span class="cf">return</span> output</span>
<span id="cb3-11"><a></a></span>
<span id="cb3-12"><a></a>    <span class="at">@staticmethod</span></span>
<span id="cb3-13"><a></a>    <span class="kw">def</span> backward(ctx, grad_output):</span>
<span id="cb3-14"><a></a>        q <span class="op">=</span> ctx.saved_tensors[<span class="dv">0</span>]</span>
<span id="cb3-15"><a></a>        <span class="co"># Calculate gradients (another triton kernel)</span></span>
<span id="cb3-16"><a></a>        <span class="cf">return</span> ...</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
</section>
<section id="some-benchmarks" class="slide level2">
<h2><code>Some benchmarks</code></h2>
<ul>
<li>A KL Divergence kernel that is currently used in <a href="https://github.com/linkedin/liger-kernel">Liger Kernel</a> written by <span class="citation" data-cites="me">@me</span></li>
</ul>
<div class="columns">
<div class="column" style="width:50%;">
<div class="quarto-figure quarto-figure-center">
<figure>
<p><img data-src="media/kl_mem.png" class="center quarto-figure quarto-figure-center"></p>
</figure>
</div>
</div><div class="column" style="width:50%;">
<div class="quarto-figure quarto-figure-center">
<figure>
<p><img data-src="media/kl_speed.png" class="center quarto-figure quarto-figure-center"></p>
</figure>
</div>
</div></div>
</section>
<section id="do-i-have-to-write-everything" class="slide level2">
<h2><code>Do I have to write everything?</code></h2>
<ul>
<li>TLDR: No</li>
<li>Many cool projects already using Triton</li>
<li>Better Integration with PyTorch and even Hugging Face 🤗</li>
<li>Liger Kernel, Unsloth AI, etc.</li>
</ul>
<div class="columns">
<div class="column" style="width:50%;">
<div class="quarto-figure quarto-figure-center">
<figure>
<p><img data-src="media/unsloth.png" class="center quarto-figure quarto-figure-center"></p>
</figure>
</div>
</div><div class="column" style="width:50%;">
<div class="quarto-figure quarto-figure-center">
<figure>
<p><img data-src="media/liger.png" class="center quarto-figure quarto-figure-center"></p>
</figure>
</div>
</div></div>
</section>
<section id="so-how-can-i-use-this-in-my-llm" class="slide level2">
<h2><code>So how can I use this in my LLM? 🚀</code></h2>
<ul>
<li>Liger Kernel is a great example, providing examples of how to integrate with Hugging Face 🤗 Trainer</li>
</ul>
<div class="sourceCode" id="cb4"><pre class="sourceCode numberSource diff number-lines code-with-copy"><code class="sourceCode diff"><span id="cb4-1"><a></a><span class="st">- from transformers import AutoModelForCausalLM</span></span>
<span id="cb4-2"><a></a><span class="va">+ from liger_kernel.transformers import AutoLigerKernelForCausalLM</span></span>
<span id="cb4-3"><a></a></span>
<span id="cb4-4"><a></a>model_path = "meta-llama/Meta-Llama-3-8B-Instruct"</span>
<span id="cb4-5"><a></a></span>
<span id="cb4-6"><a></a><span class="st">- model = AutoModelForCausalLM.from_pretrained(model_path)</span></span>
<span id="cb4-7"><a></a><span class="va">+ model = AutoLigerKernelForCausalLM.from_pretrained(model_path)</span></span>
<span id="cb4-8"><a></a></span>
<span id="cb4-9"><a></a># training/inference logic...</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
</section>
<section id="key-optimization-techniques-adapted-by-liger-kernel" class="slide level2">
<h2><code>Key Optimization Techniques adapted by Liger Kernel</code></h2>
<ul>
<li>Kernel Fusion</li>
<li>Domain-specific optimizations</li>
<li>Memory Access Patterns</li>
<li>Preemptive memory freeing</li>
</ul>
</section>
<section id="aaand-some-more-benchmarks" class="slide level2">
<h2><code>Aaand some more benchmarks 🚀</code></h2>
<div class="columns">
<div class="column" style="width:50%;">
<div class="quarto-figure quarto-figure-center">
<figure>
<p><img data-src="media/PMA.png" class="quarto-figure quarto-figure-center"></p>
</figure>
</div>
</div><div class="column" style="width:50%;">
<div class="quarto-figure quarto-figure-center">
<figure>
<p><img data-src="media/PMR.png" class="quarto-figure quarto-figure-center"></p>
</figure>
</div>
</div></div>
</section>
<section id="last-benchmark-i-promise..." class="slide level2">
<h2><code>Last benchmark I promise...</code></h2>
<p><img data-src="media/TPS.png" style="width:50.0%;height:50.0%"></p>
<div>
<p><em>Attention is all you need, so I thank you for yours!</em> 🤗</p>
</div>


</section>
    </div>
  <div class="quarto-auto-generated-content" style="display: none;">
<div class="footer footer-default">

</div>
</div></div>

  <script>window.backupDefine = window.define; window.define = undefined;</script>
  <script src="site_libs/revealjs/dist/reveal.js"></script>
  <!-- reveal.js plugins -->
  <script src="site_libs/revealjs/plugin/quarto-line-highlight/line-highlight.js"></script>
  <script src="site_libs/revealjs/plugin/pdf-export/pdfexport.js"></script>
  <script src="site_libs/revealjs/plugin/reveal-menu/menu.js"></script>
  <script src="site_libs/revealjs/plugin/reveal-menu/quarto-menu.js"></script>
  <script src="site_libs/revealjs/plugin/quarto-support/support.js"></script>
  

  <script src="site_libs/revealjs/plugin/notes/notes.js"></script>
  <script src="site_libs/revealjs/plugin/search/search.js"></script>
  <script src="site_libs/revealjs/plugin/zoom/zoom.js"></script>
  <script src="site_libs/revealjs/plugin/math/math.js"></script>
  <script>window.define = window.backupDefine; window.backupDefine = undefined;</script>

  <script>

      // Full list of configuration options available at:
      // https://revealjs.com/config/
      Reveal.initialize({
'controlsAuto': true,
'previewLinksAuto': false,
'pdfSeparateFragments': false,
'autoAnimateEasing': "ease",
'autoAnimateDuration': 1,
'autoAnimateUnmatched': true,
'jumpToSlide': true,
'menu': {"side":"left","useTextContentForMissingTitles":true,"markers":false,"loadIcons":false,"custom":[{"title":"Tools","icon":"<i class=\"fas fa-gear\"></i>","content":"<ul class=\"slide-menu-items\">\n<li class=\"slide-tool-item active\" data-item=\"0\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.fullscreen(event)\"><kbd>f</kbd> Fullscreen</a></li>\n<li class=\"slide-tool-item\" data-item=\"1\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.speakerMode(event)\"><kbd>s</kbd> Speaker View</a></li>\n<li class=\"slide-tool-item\" data-item=\"2\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.overview(event)\"><kbd>o</kbd> Slide Overview</a></li>\n<li class=\"slide-tool-item\" data-item=\"3\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.togglePdfExport(event)\"><kbd>e</kbd> PDF Export Mode</a></li>\n<li class=\"slide-tool-item\" data-item=\"4\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.toggleScrollView(event)\"><kbd>r</kbd> Scroll View Mode</a></li>\n<li class=\"slide-tool-item\" data-item=\"5\"><a href=\"#\" onclick=\"RevealMenuToolHandlers.keyboardHelp(event)\"><kbd>?</kbd> Keyboard Help</a></li>\n</ul>"}],"openButton":true},
'smaller': false,
 
        // Display controls in the bottom right corner
        controls: false,

        // Help the user learn the controls by providing hints, for example by
        // bouncing the down arrow when they first encounter a vertical slide
        controlsTutorial: false,

        // Determines where controls appear, "edges" or "bottom-right"
        controlsLayout: 'edges',

        // Visibility rule for backwards navigation arrows; "faded", "hidden"
        // or "visible"
        controlsBackArrows: 'faded',

        // Display a presentation progress bar
        progress: true,

        // Display the page number of the current slide
        slideNumber: 'c/t',

        // 'all', 'print', or 'speaker'
        showSlideNumber: 'all',

        // Add the current slide number to the URL hash so that reloading the
        // page/copying the URL will return you to the same slide
        hash: true,

        // Start with 1 for the hash rather than 0
        hashOneBasedIndex: false,

        // Flags if we should monitor the hash and change slides accordingly
        respondToHashChanges: true,

        // Push each slide change to the browser history
        history: true,

        // Enable keyboard shortcuts for navigation
        keyboard: true,

        // Enable the slide overview mode
        overview: true,

        // Disables the default reveal.js slide layout (scaling and centering)
        // so that you can use custom CSS layout
        disableLayout: false,

        // Vertical centering of slides
        center: false,

        // Enables touch navigation on devices with touch input
        touch: true,

        // Loop the presentation
        loop: false,

        // Change the presentation direction to be RTL
        rtl: false,

        // see https://revealjs.com/vertical-slides/#navigation-mode
        navigationMode: 'linear',

        // Randomizes the order of slides each time the presentation loads
        shuffle: false,

        // Turns fragments on and off globally
        fragments: true,

        // Flags whether to include the current fragment in the URL,
        // so that reloading brings you to the same fragment position
        fragmentInURL: false,

        // Flags if the presentation is running in an embedded mode,
        // i.e. contained within a limited portion of the screen
        embedded: false,

        // Flags if we should show a help overlay when the questionmark
        // key is pressed
        help: true,

        // Flags if it should be possible to pause the presentation (blackout)
        pause: true,

        // Flags if speaker notes should be visible to all viewers
        showNotes: false,

        // Global override for autoplaying embedded media (null/true/false)
        autoPlayMedia: null,

        // Global override for preloading lazy-loaded iframes (null/true/false)
        preloadIframes: null,

        // Number of milliseconds between automatically proceeding to the
        // next slide, disabled when set to 0, this value can be overwritten
        // by using a data-autoslide attribute on your slides
        autoSlide: 0,

        // Stop auto-sliding after user input
        autoSlideStoppable: true,

        // Use this method for navigation when auto-sliding
        autoSlideMethod: null,

        // Specify the average time in seconds that you think you will spend
        // presenting each slide. This is used to show a pacing timer in the
        // speaker view
        defaultTiming: null,

        // Enable slide navigation via mouse wheel
        mouseWheel: false,

        // The display mode that will be used to show slides
        display: 'block',

        // Hide cursor if inactive
        hideInactiveCursor: true,

        // Time before the cursor is hidden (in ms)
        hideCursorTime: 5000,

        // Opens links in an iframe preview overlay
        previewLinks: false,

        // Transition style (none/fade/slide/convex/concave/zoom)
        transition: 'slide',

        // Transition speed (default/fast/slow)
        transitionSpeed: 'default',

        // Transition style for full page slide backgrounds
        // (none/fade/slide/convex/concave/zoom)
        backgroundTransition: 'none',

        // Number of slides away from the current that are visible
        viewDistance: 3,

        // Number of slides away from the current that are visible on mobile
        // devices. It is advisable to set this to a lower number than
        // viewDistance in order to save resources.
        mobileViewDistance: 2,

        // The "normal" size of the presentation, aspect ratio will be preserved
        // when the presentation is scaled to fit different resolutions. Can be
        // specified using percentage units.
        width: 1050,

        height: 700,

        // Factor of the display size that should remain empty around the content
        margin: 0.1,

        math: {
          mathjax: 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.9/MathJax.js',
          config: 'TeX-AMS_HTML-full',
          tex2jax: {
            inlineMath: [['\\(','\\)']],
            displayMath: [['\\[','\\]']],
            balanceBraces: true,
            processEscapes: false,
            processRefs: true,
            processEnvironments: true,
            preview: 'TeX',
            skipTags: ['script','noscript','style','textarea','pre','code'],
            ignoreClass: 'tex2jax_ignore',
            processClass: 'tex2jax_process'
          },
        },

        // reveal.js plugins
        plugins: [QuartoLineHighlight, PdfExport, RevealMenu, QuartoSupport,

          RevealMath,
          RevealNotes,
          RevealSearch,
          RevealZoom
        ]
      });
    </script>
    <script id="quarto-html-after-body" type="application/javascript">
    window.document.addEventListener("DOMContentLoaded", function (event) {
      const toggleBodyColorMode = (bsSheetEl) => {
        const mode = bsSheetEl.getAttribute("data-mode");
        const bodyEl = window.document.querySelector("body");
        if (mode === "dark") {
          bodyEl.classList.add("quarto-dark");
          bodyEl.classList.remove("quarto-light");
        } else {
          bodyEl.classList.add("quarto-light");
          bodyEl.classList.remove("quarto-dark");
        }
      }
      const toggleBodyColorPrimary = () => {
        const bsSheetEl = window.document.querySelector("link#quarto-bootstrap");
        if (bsSheetEl) {
          toggleBodyColorMode(bsSheetEl);
        }
      }
      toggleBodyColorPrimary();  
      const tabsets =  window.document.querySelectorAll(".panel-tabset-tabby")
      tabsets.forEach(function(tabset) {
        const tabby = new Tabby('#' + tabset.id);
      });
      const isCodeAnnotation = (el) => {
        for (const clz of el.classList) {
          if (clz.startsWith('code-annotation-')) {                     
            return true;
          }
        }
        return false;
      }
      const onCopySuccess = function(e) {
        // button target
        const button = e.trigger;
        // don't keep focus
        button.blur();
        // flash "checked"
        button.classList.add('code-copy-button-checked');
        var currentTitle = button.getAttribute("title");
        button.setAttribute("title", "Copied!");
        let tooltip;
        if (window.bootstrap) {
          button.setAttribute("data-bs-toggle", "tooltip");
          button.setAttribute("data-bs-placement", "left");
          button.setAttribute("data-bs-title", "Copied!");
          tooltip = new bootstrap.Tooltip(button, 
            { trigger: "manual", 
              customClass: "code-copy-button-tooltip",
              offset: [0, -8]});
          tooltip.show();    
        }
        setTimeout(function() {
          if (tooltip) {
            tooltip.hide();
            button.removeAttribute("data-bs-title");
            button.removeAttribute("data-bs-toggle");
            button.removeAttribute("data-bs-placement");
          }
          button.setAttribute("title", currentTitle);
          button.classList.remove('code-copy-button-checked');
        }, 1000);
        // clear code selection
        e.clearSelection();
      }
      const getTextToCopy = function(trigger) {
          const codeEl = trigger.previousElementSibling.cloneNode(true);
          for (const childEl of codeEl.children) {
            if (isCodeAnnotation(childEl)) {
              childEl.remove();
            }
          }
          return codeEl.innerText;
      }
      const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
        text: getTextToCopy
      });
      clipboard.on('success', onCopySuccess);
      if (window.document.getElementById('quarto-embedded-source-code-modal')) {
        const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
          text: getTextToCopy,
          container: window.document.getElementById('quarto-embedded-source-code-modal')
        });
        clipboardModal.on('success', onCopySuccess);
      }
        var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
        var mailtoRegex = new RegExp(/^mailto:/);
          var filterRegex = new RegExp('/' + window.location.host + '/');
        var isInternal = (href) => {
            return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
        }
        // Inspect non-navigation links and adorn them if external
     	var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
        for (var i=0; i<links.length; i++) {
          const link = links[i];
          if (!isInternal(link.href)) {
            // undo the damage that might have been done by quarto-nav.js in the case of
            // links that we want to consider external
            if (link.dataset.originalHref !== undefined) {
              link.href = link.dataset.originalHref;
            }
          }
        }
      function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
        const config = {
          allowHTML: true,
          maxWidth: 500,
          delay: 100,
          arrow: false,
          appendTo: function(el) {
              return el.closest('section.slide') || el.parentElement;
          },
          interactive: true,
          interactiveBorder: 10,
          theme: 'light-border',
          placement: 'bottom-start',
        };
        if (contentFn) {
          config.content = contentFn;
        }
        if (onTriggerFn) {
          config.onTrigger = onTriggerFn;
        }
        if (onUntriggerFn) {
          config.onUntrigger = onUntriggerFn;
        }
          config['offset'] = [0,0];
          config['maxWidth'] = 700;
        window.tippy(el, config); 
      }
      const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
      for (var i=0; i<noterefs.length; i++) {
        const ref = noterefs[i];
        tippyHover(ref, function() {
          // use id or data attribute instead here
          let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
          try { href = new URL(href).hash; } catch {}
          const id = href.replace(/^#\/?/, "");
          const note = window.document.getElementById(id);
          if (note) {
            return note.innerHTML;
          } else {
            return "";
          }
        });
      }
      const findCites = (el) => {
        const parentEl = el.parentElement;
        if (parentEl) {
          const cites = parentEl.dataset.cites;
          if (cites) {
            return {
              el,
              cites: cites.split(' ')
            };
          } else {
            return findCites(el.parentElement)
          }
        } else {
          return undefined;
        }
      };
      var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
      for (var i=0; i<bibliorefs.length; i++) {
        const ref = bibliorefs[i];
        const citeInfo = findCites(ref);
        if (citeInfo) {
          tippyHover(citeInfo.el, function() {
            var popup = window.document.createElement('div');
            citeInfo.cites.forEach(function(cite) {
              var citeDiv = window.document.createElement('div');
              citeDiv.classList.add('hanging-indent');
              citeDiv.classList.add('csl-entry');
              var biblioDiv = window.document.getElementById('ref-' + cite);
              if (biblioDiv) {
                citeDiv.innerHTML = biblioDiv.innerHTML;
              }
              popup.appendChild(citeDiv);
            });
            return popup.innerHTML;
          });
        }
      }
    });
    </script>
    

</body></html>