Spaces:

avid-ml
/

indie-label

Runtime error

App Files Files Community

carolanderson commited on Mar 20

Commit

7ad15b7

•

2 Parent(s): defa0d2 3b065d6

Merge branch 'main' into docker-hf

Browse files

Files changed (14) hide show

.github/workflows/sync_fork.yml +19 -0
indie_label_svelte/public/img/anderson.jpeg +0 -0
indie_label_svelte/public/img/blili_hamelin.jpeg +0 -0
indie_label_svelte/public/img/butters.jpeg +0 -0
indie_label_svelte/public/img/landay.jpeg +0 -0
indie_label_svelte/public/img/pan.jpg +0 -0
indie_label_svelte/public/index.html +9 -0
indie_label_svelte/src/About.svelte +116 -25
indie_label_svelte/src/Auditing.svelte +0 -3
indie_label_svelte/src/ClusterResults.svelte +4 -4
indie_label_svelte/src/HypothesisPanel.svelte +9 -8
indie_label_svelte/src/MainPanel.svelte +4 -2
indie_label_svelte/src/ModelPerf.svelte +1 -1
indie_label_svelte/src/OverallResults.svelte +1 -1

.github/workflows/sync_fork.yml ADDED Viewed

	@@ -0,0 +1,19 @@

+name: Sync Fork
+on:
+  schedule:
+    - cron: '0,30 * * * *'
+  workflow_dispatch: # on button click
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: GitHub Sync to Upstream Repository
+        uses: rovast/[email protected]
+        with:
+          upstream_repo: https://github.com/StanfordHCI/indie-label.git
+          upstream_branch: main
+          downstream_branch: docker-hf
+          token: ${{ secrets.WORKFLOW_TOKEN }}

indie_label_svelte/public/img/anderson.jpeg ADDED Viewed

indie_label_svelte/public/img/blili_hamelin.jpeg ADDED Viewed

indie_label_svelte/public/img/butters.jpeg ADDED Viewed

indie_label_svelte/public/img/landay.jpeg CHANGED Viewed

indie_label_svelte/public/img/pan.jpg ADDED Viewed

indie_label_svelte/public/index.html CHANGED Viewed

@@ -19,6 +19,15 @@
 	<script defer src='/build/bundle.js'></script>
 </head>
 <body>
 </body>

 	<script defer src='/build/bundle.js'></script>
 </head>
+<!-- Google tag (gtag.js) -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-GFB1D1ZXRZ"></script>
+<script>
+  window.dataLayer = window.dataLayer || [];
+  function gtag(){dataLayer.push(arguments);}
+  gtag('js', new Date());
+  gtag('config', 'G-GFB1D1ZXRZ');
+</script>
 <body>
 </body>

indie_label_svelte/src/About.svelte CHANGED Viewed

@@ -7,6 +7,15 @@
 <div class="panel">
     <div class="panel_contents">
         <div>
             <h3>About IndieLabel</h3>
             <p class="body">
@@ -16,45 +25,62 @@
             <div class="card-container">
                 <p class="head_5">IndieLabel & the End-User Audits paper</p>
                 <p class="body">
-                    The <a href="https://github.com/StanfordHCI/indie-label" target="_blank">IndieLabel</a> tool was initially presented as a research prototype system in an academic publication called <b><i>End-User Audits: A System Empowering Communities to Lead Large-Scale Investigations of Harmful Algorithmic Behavior</i></b>, presented at CSCW 2022. The full paper is available on the <a href="https://dl.acm.org/doi/10.1145/3555625" target="_blank">ACM Digital Library</a> or via <a href="https://hci.stanford.edu/publications/2022/Lam_EndUserAudits_CSCW22.pdf" target="_blank">Stanford HCI</a>.
                 </p>
                 <p class="body">
-                    The key question underlying the End-User Audits paper is: <i>how do we enable everyday users to lead large-scale algorithm audits</i>? IndieLabel aids users in rapidly identifying where they disagree with a system's behavior with the help of a personalized recommender system model.
                 </p>
-                <p class="body">
-                    Our paper abstract summarizes our core motivation, approach, and findings:
                     "Because algorithm audits are conducted by technical experts, audits are necessarily limited to the hypotheses that experts think to test. End users hold the promise to expand this purview, as they inhabit spaces and witness algorithmic impacts that auditors do not. In pursuit of this goal, we propose end-user audits—system-scale audits led by non-technical users—and present an approach that scaffolds end users in hypothesis generation, evidence identification, and results communication. Today, performing a system-scale audit requires substantial user effort to label thousands of system outputs, so we introduce a collaborative filtering technique that leverages the algorithmic system's own disaggregated training data to project from a small number of end user labels onto the full test set. Our end-user auditing tool, IndieLabel, employs these predicted labels so that users can rapidly explore where their opinions diverge from the algorithmic system's outputs. By highlighting topic areas where the system is under-performing for the user and surfacing sets of likely error cases, the tool guides the user in authoring an audit report. In an evaluation of end-user audits on a popular comment toxicity model with 17 non-technical participants, participants both replicated issues that formal audits had previously identified and also raised previously underreported issues such as under-flagging on veiled forms of hate that perpetuate stigma and over-flagging of slurs that have been reclaimed by marginalized communities."
                 </p>
                 <p class="body">
-                    The End-User Audits work was led by Stanford PhD student <a href="http://michelle123lam.github.io/" target="_blank">Michelle Lam</a> along with co-authors <a href="https://mgordon.me/" target="_blank">Mitchell Gordon</a>, <a href="https://metaxa.net/" target="_blank">Danaë Metaxa</a>, <a href="https://sml.stanford.edu/people/jeff-hancock" target="_blank">Jeffrey Hancock</a>, <a href="https://www.landay.org/" target="_blank">James Landay</a>, and <a href="https://hci.stanford.edu/msb/" target="_blank">Michael Bernstein</a>.
                 </p>
                 <div class="image-container">
                     <div>
                         <img src="./img/lam.jpg" alt="Michelle Lam">
-                        <p><b>Michelle Lam</b><br/>PhD Student, Stanford CS</p>
                     </div>
                     <div>
                         <img src="./img/gordon.jpeg" alt="Mitchell Gordon">
-                        <p><b>Mitchell Gordon</b><br/>Incoming Asst Professor, MIT EECS</p>
                     </div>
                     <div>
                         <img src="./img/metaxa.jpg" alt="Danaë Metaxa">
-                        <p><b>Danaë Metaxa</b><br/>Asst Professor, UPenn CIS</p>
                     </div>
                     <div>
                         <img src="./img/hancock.png" alt="Jeffrey Hancock">
-                        <p><b>Jeffrey Hancock</b><br/>Professor, Stanford Comm</p>
                     </div>
                     <div>
                         <img src="./img/landay.jpeg" alt="James Landay">
-                        <p><b>James Landay</b><br/>Professor, Stanford CS</p>
                     </div>
                     <div>
                         <img src="./img/bernstein.jpeg" alt="Michael Bernstein">
-                        <p><b>Michael Bernstein</b><br/>Assoc Professor, Stanford CS</p>
                     </div>
                 </div>
             </div>
@@ -62,7 +88,7 @@
             <div class="card-container">
                 <p class="head_5">ARVA</p>
                 <p class="body">
-                    To fill in: information about ARVA and AVID.
                 </p>
             </div>
@@ -72,22 +98,41 @@
                     The IndieLabel Hugging Face deployment was made possible by a wonderful team of volunteers who worked on adapting IndieLabel for use by a general audience, connecting its reports to AVID (AI Vulnerability Database), and deploying it on Hugging Face Spaces. The team includes:
                 </p>
                 <div>
-                    <div class="image-container">
-                        <div>
                             <img src="./img/lam.jpg" alt="Michelle Lam">
-                            <p><b>Michelle Lam</b></p>
                         </div>
-                        <div>
-                            <img src="./img/temp.png" alt="Carol Anderson">
-                            <p><b>Carol Anderson</b></p>
                         </div>
-                        <div>
-                            <img src="./img/temp.png" alt="Christina Pan">
-                            <p><b>Christina Pan</b></p>
                         </div>
-                        <div>
-                            <img src="./img/temp.png" alt="Nathan Butters">
-                            <p><b>Nathan Butters</b></p>
                         </div>
                     </div>
                 </div>
@@ -102,6 +147,14 @@
         text-decoration: underline;
     }
     .head_5 {
         margin: 10px 0;
     }
@@ -131,4 +184,42 @@
         max-width: 110px;
         font-size: 11px;
     }
 </style>

 <div class="panel">
     <div class="panel_contents">
+        <div>
+            <h3>IndieLabel Tutorial</h3>
+            <p class="body">
+                Check out this 5-minute video tutorial for a quick overview of the IndieLabel tool. If you prefer a written document, you can also find a help article <a href="https://docs.google.com/document/d/1BTo8LFUriiZcSWCcQy0lel9VxGl4KZtlFnu7wAFaiu4/edit?usp=sharing" target="_blank">here</a>.
+            </p>
+            <div class="video-container">
+                <iframe src="https://www.youtube.com/embed/Je0DCDnJ6KQ?si=H1dVy-oe6PSP0QYM" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen></iframe>
+            </div>
+        </div>
         <div>
             <h3>About IndieLabel</h3>
             <p class="body">
             <div class="card-container">
                 <p class="head_5">IndieLabel & the End-User Audits paper</p>
                 <p class="body">
+                    The <a href="https://github.com/StanfordHCI/indie-label" target="_blank">IndieLabel</a> tool was initially presented as a research prototype in an academic publication called <b><i>End-User Audits: A System Empowering Communities to Lead Large-Scale Investigations of Harmful Algorithmic Behavior</i></b>. IndieLabel aids users in rapidly identifying where they disagree with a system's behavior with the help of a personalized recommender system model.
                 </p>
                 <p class="body">
+                    The key question underlying the End-User Audits paper is: <i>how do we enable everyday users to lead large-scale algorithm audits</i>? The full paper was presented at CSCW 2022 and is available on the <a href="https://dl.acm.org/doi/10.1145/3555625" target="_blank">ACM Digital Library</a> or via <a href="https://hci.stanford.edu/publications/2022/Lam_EndUserAudits_CSCW22.pdf" target="_blank">Stanford HCI</a>. The End-User Audits paper abstract summarizes our core motivation, approach, and findings:
                 </p>
+                <p class="body blockquote">
                     "Because algorithm audits are conducted by technical experts, audits are necessarily limited to the hypotheses that experts think to test. End users hold the promise to expand this purview, as they inhabit spaces and witness algorithmic impacts that auditors do not. In pursuit of this goal, we propose end-user audits—system-scale audits led by non-technical users—and present an approach that scaffolds end users in hypothesis generation, evidence identification, and results communication. Today, performing a system-scale audit requires substantial user effort to label thousands of system outputs, so we introduce a collaborative filtering technique that leverages the algorithmic system's own disaggregated training data to project from a small number of end user labels onto the full test set. Our end-user auditing tool, IndieLabel, employs these predicted labels so that users can rapidly explore where their opinions diverge from the algorithmic system's outputs. By highlighting topic areas where the system is under-performing for the user and surfacing sets of likely error cases, the tool guides the user in authoring an audit report. In an evaluation of end-user audits on a popular comment toxicity model with 17 non-technical participants, participants both replicated issues that formal audits had previously identified and also raised previously underreported issues such as under-flagging on veiled forms of hate that perpetuate stigma and over-flagging of slurs that have been reclaimed by marginalized communities."
                 </p>
                 <p class="body">
+                    The End-User Audits work was led by Stanford PhD student Michelle Lam along with co-authors Mitchell Gordon, Danaë Metaxa, Jeffrey Hancock, James Landay, and Michael Bernstein.
                 </p>
                 <div class="image-container">
                     <div>
                         <img src="./img/lam.jpg" alt="Michelle Lam">
+                        <p>
+                            <a href="http://michelle123lam.github.io" target="_blank"><b>Michelle Lam</b></a><br>
+                            PhD Student, Stanford CS
+                        </p>
                     </div>
                     <div>
                         <img src="./img/gordon.jpeg" alt="Mitchell Gordon">
+                        <p>
+                            <a href="https://mgordon.me/" target="_blank"><b>Mitchell Gordon</b></a><br/>
+                            Incoming Asst Professor, MIT EECS
+                        </p>
                     </div>
                     <div>
                         <img src="./img/metaxa.jpg" alt="Danaë Metaxa">
+                        <p>
+                            <a href="https://metaxa.net/" target="_blank"><b>Danaë Metaxa</b></a><br/>
+                            Asst Professor, UPenn CIS
+                        </p>
                     </div>
                     <div>
                         <img src="./img/hancock.png" alt="Jeffrey Hancock">
+                        <p>
+                            <a href="https://sml.stanford.edu/people/jeff-hancock" target="_blank"><b>Jeffrey Hancock</b></a><br/>
+                            Professor, Stanford Comm
+                        </p>
                     </div>
                     <div>
                         <img src="./img/landay.jpeg" alt="James Landay">
+                        <p>
+                            <a href="https://www.landay.org/" target="_blank"><b>James Landay</b></a><br/>
+                                Professor, Stanford CS
+                            </p>
                     </div>
                     <div>
                         <img src="./img/bernstein.jpeg" alt="Michael Bernstein">
+                        <p>
+                            <a href="https://hci.stanford.edu/msb/" target="_blank"><b>Michael Bernstein</b></a><br/>
+                            Assoc Professor, Stanford CS
+                        </p>
                     </div>
                 </div>
             </div>
             <div class="card-container">
                 <p class="head_5">ARVA</p>
                 <p class="body">
+                    The <a href="https://avidml.org/arva/" target="_blank">AI Risk and Vulnerability Alliance (ARVA)</a> is a nonprofit organization focused on making AI safer for everyone. Our mission is to empower communities to recognize, diagnose, and manage vulnerabilities in AI that affects them. Our flagship project is the <a href="https://avidml.org" target="_blank">AI Vulnerability Database (AVID)</a>, is an open-source knowledge base of failure modes for AI models, datasets, and systems.
                 </p>
             </div>
                     The IndieLabel Hugging Face deployment was made possible by a wonderful team of volunteers who worked on adapting IndieLabel for use by a general audience, connecting its reports to AVID (AI Vulnerability Database), and deploying it on Hugging Face Spaces. The team includes:
                 </p>
                 <div>
+                    <div class="image-container-wide">
+                        <div class="image-wide">
                             <img src="./img/lam.jpg" alt="Michelle Lam">
+                            <p>
+                                <a href="http://michelle123lam.github.io" target="_blank"><b>Michelle Lam</b></a><br>
+                                Michelle Lam is a PhD Candidate at Stanford University in the HCI Group. Her research focuses on building systems that empower everyday users to surface their expertise to design and evaluate AI systems.
+                            </p>
+                        </div>
+                        <div class="image-wide">
+                            <img src="./img/anderson.jpeg" alt="Carol Anderson">
+                            <p>
+                                <a href="https://www.carol-anderson.com/" target="_blank"><b>Carol Anderson</b></a><br>
+                                Carol Anderson is a data scientist and machine learning practitioner with expertise in natural language processing (NLP), biological data, and AI ethics. She serves as AVID’s machine learning lead.
+                            </p>
                         </div>
+                        <div class="image-wide">
+                            <img src="./img/pan.jpg" alt="Christina Pan">
+                            <p>
+                                <a href="https://www.christinaapan.com/" target="_blank"><b>Christina Pan</b></a><br>
+                                Christina Pan started her career building machine learning (ML) models at Google, which inspired her passion for design thinking and AI ethics.
+                            </p>
                         </div>
+                        <div class="image-wide">
+                            <img src="./img/butters.jpeg" alt="Nathan Butters">
+                            <p>
+                                <b>Nathan Butters</b><br>
+                                Nathan Butters is a product manager in the Office of Ethical and Humane Use at Salesforce. He is a cofounder of the AI Risk and Vulnerability Alliance (ARVA).
+                            </p>
                         </div>
+                        <div class="image-wide">
+                            <img src="./img/blili_hamelin.jpeg" alt="Borhane Blili-Hamelin">
+                            <p>
+                                <a href="https://borhane.xyz/" target="_blank"><b>Borhane Blili-Hamelin</b></a><br>
+                                Borhane Blili-Hamelin is an ethicist, researcher and AI risk management consultant. He is an officer at the AI Risk and Vulnerability Alliance (ARVA), an affiliate at Data & Society, and a senior consultant at BABL AI.
+                            </p>
                         </div>
                     </div>
                 </div>
         text-decoration: underline;
     }
+    .panel {
+        margin: 0 40px;
+    }
+    .blockquote {
+        margin: 0 40px;
+    }
     .head_5 {
         margin: 10px 0;
     }
         max-width: 110px;
         font-size: 11px;
     }
+    .image-container-wide {
+        margin: 20px 0;
+        display: flex;
+        flex-direction: column;
+        justify-content: center;
+        gap: 20px;
+    }
+    .image-wide {
+        display: flex;
+        flex-direction: row;
+        align-items: center;
+        gap: 10px;
+    }
+    .image-container-wide img {
+        max-width: 110px; /* Adjust the maximum width of images */
+        height: 110px;
+        border-radius: 50%;
+        display: block;
+    }
+    .video-container {
+        position: relative;
+        padding-bottom: 10px;
+        padding-top: 10px;
+        height: 40vh;
+        overflow: hidden;
+    }
+    .video-container iframe {
+        position: absolute;
+        top: 0;
+        left: 0;
+        width: 100%;
+        height: 100%;
+    }
 </style>

indie_label_svelte/src/Auditing.svelte CHANGED Viewed

@@ -293,9 +293,6 @@
                 {#if audit_results}
                     <OverallResults
                         data={audit_results}
-                        clusters={clusters}
-                        personalized_model={personalized_model}
-                        cluster={topic}
                     />
                 {/if}
             {:catch error}

                 {#if audit_results}
                     <OverallResults
                         data={audit_results}
                     />
                 {/if}
             {:catch error}

indie_label_svelte/src/ClusterResults.svelte CHANGED Viewed

@@ -32,7 +32,7 @@
     export let show_checkboxes = true;
     export let table_width_pct = 80;
     export let rowsPerPage = 10;
-    export let evidence;
     export let table_id;
     export let use_model = true;
     export let show_agree_disagree = false;
@@ -232,7 +232,7 @@
         <div class="row">
             <div class="col s8">
                 <VegaLite
-                    {cluster_overview_data}
                     spec={cluster_overview_spec}
                     bind:view={cluster_overview_view}
                 />
@@ -302,7 +302,7 @@
                     <Cell>
                         System<br>decision<br>
                         {#if show_checkboxes}
-                        <span style="font-size:12px; max-width:125px">White = Non-toxic, <br>Grey = Toxic</span>
                         {/if}
                     </Cell>
                     {#if show_num_ratings}
@@ -314,7 +314,7 @@
                     <Cell>
                         Potential error<br>type<br>
                         {#if show_checkboxes}
-                        <span style="font-size:12px; max-width:125px">Darker red = Greater <br>potential system error</span>
                         {/if}
                     </Cell>

     export let show_checkboxes = true;
     export let table_width_pct = 80;
     export let rowsPerPage = 10;
+    export let evidence = null;
     export let table_id;
     export let use_model = true;
     export let show_agree_disagree = false;
         <div class="row">
             <div class="col s8">
                 <VegaLite
+                    data={cluster_overview_data}
                     spec={cluster_overview_spec}
                     bind:view={cluster_overview_view}
                 />
                     <Cell>
                         System<br>decision<br>
                         {#if show_checkboxes}
+                        <span style="font-size:9px; max-width:125px">White = Non-toxic, <br>Grey = Toxic</span>
                         {/if}
                     </Cell>
                     {#if show_num_ratings}
                     <Cell>
                         Potential error<br>type<br>
                         {#if show_checkboxes}
+                        <span style="font-size:9px; max-width:125px">Darker red = Greater <br>potential system error</span>
                         {/if}
                     </Cell>

indie_label_svelte/src/HypothesisPanel.svelte CHANGED Viewed

@@ -248,8 +248,8 @@
         <div class="panel_header">
             <div class="panel_header_content">
                 <div class="page_header">
-                    <img src="/logo.png" style="height: 50px; padding: 0px 20px;" alt="IndieLabel" />
-                    <Button class="user_button" color="secondary" style="margin: 12px 10px;" >
                         <Label>User: {cur_user}</Label>
                     </Button>
                 </div>
@@ -259,7 +259,7 @@
                         on:click={() => (open = !open)}
                         color="primary"
                         disabled={model == null}
-                        style="float: right; padding: 10px; margin-right: 10px;"
                     >
                         {#if open}
                         <Label>Close</Label>
@@ -584,18 +584,17 @@
     :global(.mdc-button.user_button span) {
         text-overflow: ellipsis;
-        white-space: nowrap;
         overflow: hidden;
     }
     .page_header {
         width: 100%;
         background: #e3d6fd;
-        /* padding: 21px 0; */
-        /* border-bottom: 1px solid #e3d6fd; */
         padding: 10.5px 0;
         position: relative;
-        display: inline-block;
     }
     .page_header:before {
@@ -608,9 +607,11 @@
     }
     .hypotheses_header {
-        display: inline-block;
         width: 100%;
         padding: 10px 0;
         vertical-align: middle;
     }
 </style>

         <div class="panel_header">
             <div class="panel_header_content">
                 <div class="page_header">
+                    <img src="/logo.png" style="height: 50px;" alt="IndieLabel" />
+                    <Button class="user_button" color="secondary" style="margin: 0 5px; padding: 0 5px;" >
                         <Label>User: {cur_user}</Label>
                     </Button>
                 </div>
                         on:click={() => (open = !open)}
                         color="primary"
                         disabled={model == null}
+                        style="float: right; padding: 0 5px; margin: 0 5px; max-width: 200px;"
                     >
                         {#if open}
                         <Label>Close</Label>
     :global(.mdc-button.user_button span) {
         text-overflow: ellipsis;
         overflow: hidden;
     }
     .page_header {
         width: 100%;
         background: #e3d6fd;
         padding: 10.5px 0;
         position: relative;
+        display: flex;
+        align-items: center;
+        justify-content: space-around;
     }
     .page_header:before {
     }
     .hypotheses_header {
         width: 100%;
         padding: 10px 0;
         vertical-align: middle;
+        display: flex;
+        align-items: center;
+        justify-content: space-around;
     }
 </style>

indie_label_svelte/src/MainPanel.svelte CHANGED Viewed

@@ -7,18 +7,20 @@
 	import TabBar from "@smui/tab-bar";
 	import { Icon } from "@smui/common";
-	export let model;
 	export let error_type;
 	export let cur_user;
 	// Handle routing
-	let active = "labeling";
     let searchParams = new URLSearchParams(window.location.search);
     let tab = searchParams.get("tab");
 	if (tab == "auditing") {
 		active = "auditing";
 	} else if (tab == "about") {
 		active = "about";
 	}
 </script>

 	import TabBar from "@smui/tab-bar";
 	import { Icon } from "@smui/common";
+	export let model = null;
 	export let error_type;
 	export let cur_user;
 	// Handle routing
+	let active = "about";  // default tab on load
     let searchParams = new URLSearchParams(window.location.search);
     let tab = searchParams.get("tab");
 	if (tab == "auditing") {
 		active = "auditing";
 	} else if (tab == "about") {
 		active = "about";
+	} else if (tab == "labeling") {
+		active = "labeling";
 	}
 </script>

indie_label_svelte/src/ModelPerf.svelte CHANGED Viewed

@@ -32,7 +32,7 @@
     <div>
         <!-- Performance visualization -->
         <div>
-            <VegaLite {perf_plot_data} spec={perf_plot_spec} bind:view={perf_plot_view}/>
         </div>
     </div>
 </div>

     <div>
         <!-- Performance visualization -->
         <div>
+            <VegaLite data={perf_plot_data} spec={perf_plot_spec} bind:view={perf_plot_view}/>
         </div>
     </div>
 </div>

indie_label_svelte/src/OverallResults.svelte CHANGED Viewed

@@ -65,7 +65,7 @@
         {/if}
     <div class="row">
         <div class="col s8">
-            <VegaLite {topic_overview_data} spec={topic_overview_spec} bind:view={topic_overview_view}/>
         </div>
     </div>

         {/if}
     <div class="row">
         <div class="col s8">
+            <VegaLite data={topic_overview_data} spec={topic_overview_spec} bind:view={topic_overview_view}/>
         </div>
     </div>