Hugging Face's logo

Spaces:
avid-ml
/
indie-label
Runtime error

App Files Community
Michelle Lam commited on
Commit
1882b75
•
1 Parent(s): 9fa8307

Adds interaction flow to formally send reports to AVID (reviewing reports and adding additional info); puts print statements behind debug flag

Browse files
Files changed (4) hide show
  1. audit_utils.py +31 -3
  2. indie_label_svelte/src/HypothesisPanel.svelte +270 -291
  3. indie_label_svelte/src/SubmitReportDialog.svelte +120 -0
  4. server.py +42 -12
audit_utils.py CHANGED
@@ -317,6 +317,25 @@ def get_grp_model_labels(n_label_per_bin, score_bins, grp_ids):
317
 
318
  ########################################
319
  # SAVE_REPORT utils
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
320
  # Convert indielabel json to AVID json format.
321
  # See the AVID format in https://avidml.org/avidtools/reference/report
322
  #
@@ -330,7 +349,7 @@ def get_grp_model_labels(n_label_per_bin, score_bins, grp_ids):
330
  # user_rating personal_model_score 0.92
331
  # user_decision user_decision "Non-toxic"
332
  # Note that this is at the individual report level.
333
- def convert_indie_label_json_to_avid_json(indie_label_json):
334
 
335
  # Setting up the structure with a dict to enable programmatic additions
336
  avid_json_dict = {
@@ -379,12 +398,21 @@ def convert_indie_label_json_to_avid_json(indie_label_json):
379
  "taxonomy_version": "0.2"
380
  }
381
  },
382
- "credit": None,
383
  "reported_date": "" # Leaving empty so that it can be dynamically filled in
384
  }
385
 
386
- avid_json_dict["description"] = indie_label_json["text_entry"]
387
  avid_json_dict["reported_date"] = str(date.today())
 
 
 
 
 
 
 
 
 
388
  for e in indie_label_json["evidence"]:
389
  curr_metric = {}
390
  curr_metric["name"] = "Perspective API"
 
317
 
318
  ########################################
319
  # SAVE_REPORT utils
320
+
321
+ # Convert the SEP field selection from the UI to the SEP enum value
322
+ def get_sep_enum(sep_selection):
323
+ if sep_selection == "Adversarial Example":
324
+ return "S0403: Adversarial Example"
325
+ elif sep_selection == "Accuracy":
326
+ return "P0204: Accuracy"
327
+ elif sep_selection == "Bias/Discrimination":
328
+ return "E0100: Bias/ Discrimination"
329
+ else:
330
+ return "P0200: Model issues"
331
+
332
+ # Format the description for the report including the provided title, error type, and text entry field ("Summary/Suggestions" text box)
333
+ def format_description(indie_label_json):
334
+ title = indie_label_json["title"]
335
+ error_type = indie_label_json["error_type"]
336
+ text_entry = indie_label_json["text_entry"]
337
+ return f"Title: {title}\nError Type: {error_type}\nSummary/Suggestions: {text_entry}"
338
+
339
  # Convert indielabel json to AVID json format.
340
  # See the AVID format in https://avidml.org/avidtools/reference/report
341
  #
 
349
  # user_rating personal_model_score 0.92
350
  # user_decision user_decision "Non-toxic"
351
  # Note that this is at the individual report level.
352
+ def convert_indie_label_json_to_avid_json(indie_label_json, cur_user, email, sep_selection):
353
 
354
  # Setting up the structure with a dict to enable programmatic additions
355
  avid_json_dict = {
 
398
  "taxonomy_version": "0.2"
399
  }
400
  },
401
+ "credit": "", # Leaving empty so that credit can be assigned
402
  "reported_date": "" # Leaving empty so that it can be dynamically filled in
403
  }
404
 
405
+ avid_json_dict["description"] = format_description(indie_label_json)
406
  avid_json_dict["reported_date"] = str(date.today())
407
+ # Assign credit to email if provided, otherwise default to randomly assigned username
408
+ if email != "":
409
+ avid_json_dict["credit"] = email
410
+ else:
411
+ avid_json_dict["credit"] = cur_user
412
+
413
+ sep_enum = get_sep_enum(sep_selection)
414
+ avid_json_dict["impact"]["avid"]["sep_view"] = [sep_enum]
415
+
416
  for e in indie_label_json["evidence"]:
417
  curr_metric = {}
418
  curr_metric["name"] = "Perspective API"
indie_label_svelte/src/HypothesisPanel.svelte CHANGED
@@ -1,6 +1,7 @@
1
  <script lang="ts">
2
  import { onMount } from "svelte";
3
  import ClusterResults from "./ClusterResults.svelte";
 
4
 
5
  import Button, { Label } from "@smui/button";
6
  import Textfield from '@smui/textfield';
@@ -31,6 +32,8 @@
31
  topic_chosen.subscribe(value => {
32
  cur_topic = value;
33
  });
 
 
34
 
35
  // Handle routing
36
  let searchParams = new URLSearchParams(window.location.search);
@@ -202,333 +205,309 @@
202
  // Save current error type
203
  async function updateErrorType() {
204
  // Update error type on main page to be the selected error type
205
- // error_type.update((value) => cur_error_type);
206
- // selected["error_type"] = cur_error_type;
207
  editErrorType = false;
208
  }
209
 
 
 
 
 
 
 
 
 
 
210
  </script>
211
 
212
- <div class="hypothesis_panel">
213
- <div class="panel_header">
214
- <div class="panel_header_content">
215
- <div class="page_header">
216
- <img src="/logo.png" style="height: 50px; padding: 0px 20px;" alt="IndieLabel" />
217
- <Button class="user_button" color="secondary" style="margin: 12px 10px;" >
218
- <Label>User: {cur_user}</Label>
219
- </Button>
220
- </div>
221
- <div class="hypotheses_header">
222
- <h5 style="float: left; margin: 0; padding: 5px 20px;">Your Audit Reports</h5>
223
- <Button
224
- on:click={() => (open = !open)}
225
- color="primary"
226
- disabled={model == null}
227
- style="float: right; padding: 10px; margin-right: 10px;"
228
- >
229
- {#if open}
230
- <Label>Close</Label>
231
- {:else}
232
- {#key unfinished_count}
233
- <Label>Unfinished reports ({unfinished_count})</Label>
234
- {/key}
235
- {/if}
236
- </Button>
 
 
 
 
 
 
 
 
 
237
  </div>
238
  </div>
239
- </div>
240
 
241
- {#if model == null}
242
- <div class="panel_contents">
243
- <p>You can start to author audit reports in this panel after you've trained your personalized model in the "Labeling" tab.</p>
244
- </div>
245
- {:else}
246
- <div class="panel_contents">
247
- <!-- Drawer -->
248
- {#await promise}
249
- <div class="app_loading_fullwidth">
250
- <LinearProgress indeterminate />
251
- </div>
252
- {:then reports}
253
- {#if reports}
254
- <div class="drawer-container">
255
- {#key open}
256
- <Drawer variant="dismissible" bind:open>
257
- <Header>
258
- <Title>Your Reports</Title>
259
- <Subtitle>Select a report to view.</Subtitle>
260
- </Header>
261
- <Content>
262
- <List twoLine>
263
- {#each reports as report}
264
- <Item
265
- href="javascript:void(0)"
266
- on:click={() => setActive(report)}
267
- activated={selected === report}
268
- >
269
- {#if report["complete_status"]}
270
- <Graphic class="material-icons" aria-hidden="true">task_alt</Graphic>
271
- {:else}
272
- <Graphic class="material-icons" aria-hidden="true">radio_button_unchecked</Graphic>
273
- {/if}
274
- <Text>
275
- <PrimaryText>
276
- {report["title"]}
277
- </PrimaryText>
278
- <SecondaryText>
279
- {report["error_type"]}
280
- </SecondaryText>
281
- </Text>
282
- </Item>
283
- {/each}
284
- </List>
285
- </Content>
286
- </Drawer>
287
- {/key}
288
- <AppContent class="app-content">
289
- <main class="main-content">
290
- {#if selected}
291
- <div class="head_6_highlight">
292
- Current Report
293
- </div>
294
- <div class="panel_contents2">
295
- <!-- Title -->
296
- <div class="spacing_vert">
297
- <div class="edit_button_row">
298
- {#if editTitle}
299
- <div class="edit_button_row_input">
300
- <Textfield
301
- bind:value={selected["title"]}
302
- label="Your report title"
303
- input$rows={4}
304
- textarea
305
- variant="outlined"
306
- style="width: 100%;"
307
- helperLine$style="width: 100%;"
308
- />
309
- </div>
310
- <div>
311
- <IconButton class="material-icons grey_button" size="button" on:click={() => (editTitle = false)}>
312
- check
313
- </IconButton>
314
- </div>
315
- {:else}
316
- {#if selected["title"] != ""}
317
- <div class="head_5">
318
- {selected["title"]}
319
- </div>
320
  {:else}
321
- <div class="grey_text">Enter a report title</div>
322
  {/if}
323
-
324
- <div>
325
- <IconButton class="material-icons grey_button" size="button" on:click={() => (editTitle = true)}>
326
- create
327
- </IconButton>
328
- </div>
329
- {/if}
330
- </div>
 
 
 
 
 
 
 
 
 
 
 
331
  </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
332
 
333
- <!-- Error type -->
334
- <div class="spacing_vert_40">
335
- <div class="head_6">
336
- <b>Error Type</b>
 
 
 
337
  </div>
338
- <div class="edit_button_row">
339
- {#if editErrorType}
340
- <div>
341
- {#each error_type_options as e}
342
- <div style="display: flex; align-items: center;">
343
- <!-- <Wrapper rich>
 
 
 
 
 
344
  <FormField>
345
  <Radio bind:group={selected["error_type"]} value={e.opt} on:change={updateErrorType} color="secondary" />
346
  <span slot="label">
347
- {e.opt}
348
- <IconButton class="material-icons" size="button" disabled>help_outline</IconButton>
349
  </span>
350
  </FormField>
351
- <HelpTooltip text={e.help} />
352
- </Wrapper> -->
353
-
354
- <FormField>
355
- <Radio bind:group={selected["error_type"]} value={e.opt} on:change={updateErrorType} color="secondary" />
356
- <span slot="label">
357
- <b>{e.opt}</b> {e.descr}
358
- </span>
359
- </FormField>
360
  </div>
361
- {/each}
362
- </div>
363
- <!-- <div>
364
- <IconButton class="material-icons grey_button" size="button" on:click={() => (editErrorType = false)}>
365
- check
366
- </IconButton>
367
- </div> -->
368
- {:else}
369
- {#if selected["error_type"] != ""}
370
  <div>
371
- <p>{selected["error_type"]}</p>
 
 
372
  </div>
373
- {:else}
374
- <div class="grey_text">Select an error type</div>
375
  {/if}
376
-
377
- <div>
378
- <IconButton class="material-icons grey_button" size="button" on:click={() => (editErrorType = true)}>
379
- create
380
- </IconButton>
381
- </div>
382
- {/if}
383
- </div>
384
- </div>
385
-
386
- <!-- Evidence -->
387
- <div class="spacing_vert_40">
388
- <div class="head_6">
389
- <b>Evidence</b>
390
  </div>
391
- {#key cur_open_evidence}
392
- <div>
393
- {#if cur_open_evidence.length > 0}
394
- <ClusterResults
395
- cluster={cur_topic}
396
- model={model}
397
- data={{"cluster_comments": cur_open_evidence}}
398
- show_vis={false}
399
- show_checkboxes={false}
400
- table_width_pct={100}
401
- rowsPerPage={25}
402
- table_id={"panel"}
403
- />
404
- {:else}
405
- <p class="grey_text">
406
- Add examples from the main panel to see them here!
407
- </p>
408
- {/if}
 
 
 
 
 
 
 
 
409
  </div>
410
- {/key}
411
- </div>
412
 
413
- <div class="spacing_vert_60">
414
- <div class="head_6">
415
- <b>Summary/Suggestions</b>
416
- </div>
417
- <div class="spacing_vert">
418
- <Textfield
419
- style="width: 100%;"
420
- helperLine$style="width: 100%;"
421
- input$rows={8}
422
- textarea
423
- bind:value={selected["text_entry"]}
424
- label="My current hunch is that..."
425
- >
426
- </Textfield>
 
 
427
  </div>
428
-
429
- </div>
430
 
431
- <div class="spacing_vert_40">
432
- <div class="head_6">
433
- <b>Mark report as complete?</b>
434
- <FormField>
435
- <Checkbox checked={selected["complete_status"]} on:change={handleMarkComplete} />
436
- </FormField>
 
 
437
  </div>
438
-
439
  </div>
440
- </div>
441
- {/if}
442
- </main>
443
- </AppContent>
444
- </div>
445
- {/if}
446
- {:catch error}
447
- <p style="color: red">{error.message}</p>
448
- {/await}
449
- </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
450
 
451
- <div class="panel_footer">
452
- <div class="panel_footer_contents">
453
-
454
-
455
- <Button
456
- on:click={handleNewReport}
457
- variant="outlined"
458
- color="secondary"
459
- style=""
460
- >
461
- <Label>New</Label>
462
- </Button>
463
-
464
- <Button
465
- on:click={handleDeleteReport}
466
- variant="outlined"
467
- color="secondary"
468
- style=""
469
- >
470
- <Label>Delete</Label>
471
- </Button>
472
-
473
- <Button
474
- on:click={handleSaveReport}
475
- variant="outlined"
476
- color="secondary"
477
- >
478
- <Label>Save</Label>
479
- </Button>
480
-
481
- <div>
482
- <span style="color: grey"><i>Last saved:
483
- {#await promise_save}
484
- <CircularProgress style="height: 32px; width: 32px;" indeterminate />
485
- {:then result}
486
- {#if result}
487
- {new Date().toLocaleTimeString()}
488
- {:else}
489
- —
490
- {/if}
491
- {:catch error}
492
- <p style="color: red">{error.message}</p>
493
- {/await}
494
- </i></span>
495
  </div>
496
  </div>
 
497
  </div>
498
- {/if}
499
- <!-- TEMP -->
500
- <!-- {#key model}
501
- <div>Model: {model}</div>
502
- {/key} -->
503
  </div>
504
 
505
  <style>
506
- /* Drawer */
507
- /* .drawer-container {
508
- position: relative;
509
- display: flex;
510
- height: 350px;
511
- max-width: 600px;
512
- border: 1px solid
513
- var(--mdc-theme-text-hint-on-background, rgba(0, 0, 0, 0.1));
514
- overflow: hidden;
515
- z-index: 0;
516
- }
517
-
518
- * :global(.app-content) {
519
- flex: auto;
520
- overflow: auto;
521
- position: relative;
522
- flex-grow: 1;
523
- }
524
-
525
- .main-content {
526
- overflow: auto;
527
- padding: 16px;
528
- height: 100%;
529
- box-sizing: border-box;
530
- } */
531
-
532
  .panel_contents {
533
  padding: 0 20px;
534
  overflow-y: auto;
 
1
  <script lang="ts">
2
  import { onMount } from "svelte";
3
  import ClusterResults from "./ClusterResults.svelte";
4
+ import SubmitReportDialog from "./SubmitReportDialog.svelte";
5
 
6
  import Button, { Label } from "@smui/button";
7
  import Textfield from '@smui/textfield';
 
32
  topic_chosen.subscribe(value => {
33
  cur_topic = value;
34
  });
35
+ // Handle submit report dialog
36
+ let submit_dialog_open = false;
37
 
38
  // Handle routing
39
  let searchParams = new URLSearchParams(window.location.search);
 
205
  // Save current error type
206
  async function updateErrorType() {
207
  // Update error type on main page to be the selected error type
 
 
208
  editErrorType = false;
209
  }
210
 
211
+ let promise_submit = Promise.resolve(null);
212
+ function handleSubmitReport() {
213
+ promise_submit = submitReport();
214
+ }
215
+ async function submitReport() {
216
+ submit_dialog_open = true;
217
+ return true;
218
+ }
219
+
220
  </script>
221
 
222
+ <div>
223
+ {#await promise_submit}
224
+ <CircularProgress style="height: 32px; width: 32px;" indeterminate />
225
+ {:then}
226
+ <SubmitReportDialog bind:open={submit_dialog_open} cur_user={cur_user} all_reports={all_reports}/>
227
+ {:catch error}
228
+ <p style="color: red">{error.message}</p>
229
+ {/await}
230
+ <div class="hypothesis_panel">
231
+ <div class="panel_header">
232
+ <div class="panel_header_content">
233
+ <div class="page_header">
234
+ <img src="/logo.png" style="height: 50px; padding: 0px 20px;" alt="IndieLabel" />
235
+ <Button class="user_button" color="secondary" style="margin: 12px 10px;" >
236
+ <Label>User: {cur_user}</Label>
237
+ </Button>
238
+ </div>
239
+ <div class="hypotheses_header">
240
+ <h5 style="float: left; margin: 0; padding: 5px 20px;">Your Audit Reports</h5>
241
+ <Button
242
+ on:click={() => (open = !open)}
243
+ color="primary"
244
+ disabled={model == null}
245
+ style="float: right; padding: 10px; margin-right: 10px;"
246
+ >
247
+ {#if open}
248
+ <Label>Close</Label>
249
+ {:else}
250
+ {#key unfinished_count}
251
+ <Label>Unfinished reports ({unfinished_count})</Label>
252
+ {/key}
253
+ {/if}
254
+ </Button>
255
+ </div>
256
  </div>
257
  </div>
 
258
 
259
+ {#if model == null}
260
+ <div class="panel_contents">
261
+ <p>You can start to author audit reports in this panel after you've trained your personalized model in the "Labeling" tab.</p>
262
+ </div>
263
+ {:else}
264
+ <div class="panel_contents">
265
+ <!-- Drawer -->
266
+ {#await promise}
267
+ <div class="app_loading_fullwidth">
268
+ <LinearProgress indeterminate />
269
+ </div>
270
+ {:then reports}
271
+ {#if reports}
272
+ <div class="drawer-container">
273
+ {#key open}
274
+ <Drawer variant="dismissible" bind:open>
275
+ <Header>
276
+ <Title>Your Reports</Title>
277
+ <Subtitle>Select a report to view.</Subtitle>
278
+ </Header>
279
+ <Content>
280
+ <List twoLine>
281
+ {#each reports as report}
282
+ <Item
283
+ href="javascript:void(0)"
284
+ on:click={() => setActive(report)}
285
+ activated={selected === report}
286
+ >
287
+ {#if report["complete_status"]}
288
+ <Graphic class="material-icons" aria-hidden="true">task_alt</Graphic>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
289
  {:else}
290
+ <Graphic class="material-icons" aria-hidden="true">radio_button_unchecked</Graphic>
291
  {/if}
292
+ <Text>
293
+ <PrimaryText>
294
+ {report["title"]}
295
+ </PrimaryText>
296
+ <SecondaryText>
297
+ {report["error_type"]}
298
+ </SecondaryText>
299
+ </Text>
300
+ </Item>
301
+ {/each}
302
+ </List>
303
+ </Content>
304
+ </Drawer>
305
+ {/key}
306
+ <AppContent class="app-content">
307
+ <main class="main-content">
308
+ {#if selected}
309
+ <div class="head_6_highlight">
310
+ Current Report
311
  </div>
312
+ <div class="panel_contents2">
313
+ <!-- Title -->
314
+ <div class="spacing_vert">
315
+ <div class="edit_button_row">
316
+ {#if editTitle}
317
+ <div class="edit_button_row_input">
318
+ <Textfield
319
+ bind:value={selected["title"]}
320
+ label="Your report title"
321
+ input$rows={4}
322
+ textarea
323
+ variant="outlined"
324
+ style="width: 100%;"
325
+ helperLine$style="width: 100%;"
326
+ />
327
+ </div>
328
+ <div>
329
+ <IconButton class="material-icons grey_button" size="button" on:click={() => (editTitle = false)}>
330
+ check
331
+ </IconButton>
332
+ </div>
333
+ {:else}
334
+ {#if selected["title"] != ""}
335
+ <div class="head_5">
336
+ {selected["title"]}
337
+ </div>
338
+ {:else}
339
+ <div class="grey_text">Enter a report title</div>
340
+ {/if}
341
 
342
+ <div>
343
+ <IconButton class="material-icons grey_button" size="button" on:click={() => (editTitle = true)}>
344
+ create
345
+ </IconButton>
346
+ </div>
347
+ {/if}
348
+ </div>
349
  </div>
350
+
351
+ <!-- Error type -->
352
+ <div class="spacing_vert_40">
353
+ <div class="head_6">
354
+ <b>Error Type</b>
355
+ </div>
356
+ <div class="edit_button_row">
357
+ {#if editErrorType}
358
+ <div>
359
+ {#each error_type_options as e}
360
+ <div style="display: flex; align-items: center;">
361
  <FormField>
362
  <Radio bind:group={selected["error_type"]} value={e.opt} on:change={updateErrorType} color="secondary" />
363
  <span slot="label">
364
+ <b>{e.opt}</b> {e.descr}
 
365
  </span>
366
  </FormField>
367
+ </div>
368
+ {/each}
369
+ </div>
370
+ {:else}
371
+ {#if selected["error_type"] != ""}
372
+ <div>
373
+ <p>{selected["error_type"]}</p>
 
 
374
  </div>
375
+ {:else}
376
+ <div class="grey_text">Select an error type</div>
377
+ {/if}
378
+
 
 
 
 
 
379
  <div>
380
+ <IconButton class="material-icons grey_button" size="button" on:click={() => (editErrorType = true)}>
381
+ create
382
+ </IconButton>
383
  </div>
 
 
384
  {/if}
385
+ </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
386
  </div>
387
+
388
+ <!-- Evidence -->
389
+ <div class="spacing_vert_40">
390
+ <div class="head_6">
391
+ <b>Evidence</b>
392
+ </div>
393
+ {#key cur_open_evidence}
394
+ <div>
395
+ {#if cur_open_evidence.length > 0}
396
+ <ClusterResults
397
+ cluster={cur_topic}
398
+ model={model}
399
+ data={{"cluster_comments": cur_open_evidence}}
400
+ show_vis={false}
401
+ show_checkboxes={false}
402
+ table_width_pct={100}
403
+ rowsPerPage={25}
404
+ table_id={"panel"}
405
+ />
406
+ {:else}
407
+ <p class="grey_text">
408
+ Add examples from the main panel to see them here!
409
+ </p>
410
+ {/if}
411
+ </div>
412
+ {/key}
413
  </div>
 
 
414
 
415
+ <div class="spacing_vert_60">
416
+ <div class="head_6">
417
+ <b>Summary/Suggestions</b>
418
+ </div>
419
+ <div class="spacing_vert">
420
+ <Textfield
421
+ style="width: 100%;"
422
+ helperLine$style="width: 100%;"
423
+ input$rows={8}
424
+ textarea
425
+ bind:value={selected["text_entry"]}
426
+ label="My current hunch is that..."
427
+ >
428
+ </Textfield>
429
+ </div>
430
+
431
  </div>
 
 
432
 
433
+ <div class="spacing_vert_40">
434
+ <div class="head_6">
435
+ <b>Mark report as complete?</b>
436
+ <FormField>
437
+ <Checkbox checked={selected["complete_status"]} on:change={handleMarkComplete} />
438
+ </FormField>
439
+ </div>
440
+
441
  </div>
 
442
  </div>
443
+ {/if}
444
+ </main>
445
+ </AppContent>
446
+ </div>
447
+ {/if}
448
+ {:catch error}
449
+ <p style="color: red">{error.message}</p>
450
+ {/await}
451
+ </div>
452
+
453
+ <div class="panel_footer">
454
+ <div class="panel_footer_contents">
455
+ <Button
456
+ on:click={handleNewReport}
457
+ variant="outlined"
458
+ color="secondary"
459
+ style=""
460
+ >
461
+ <Label>New</Label>
462
+ </Button>
463
+
464
+ <!-- <Button
465
+ on:click={handleDeleteReport}
466
+ variant="outlined"
467
+ color="secondary"
468
+ style=""
469
+ >
470
+ <Label>Delete</Label>
471
+ </Button> -->
472
+
473
+ <Button
474
+ on:click={handleSaveReport}
475
+ variant="outlined"
476
+ color="secondary"
477
+ >
478
+ <Label>Save</Label>
479
+ </Button>
480
 
481
+ <Button
482
+ on:click={handleSubmitReport}
483
+ variant="outlined"
484
+ color="secondary"
485
+ >
486
+ <Label>Send Reports</Label>
487
+ </Button>
488
+
489
+ <div>
490
+ <span style="color: grey"><i>Last saved:
491
+ {#await promise_save}
492
+ <CircularProgress style="height: 32px; width: 32px;" indeterminate />
493
+ {:then result}
494
+ {#if result}
495
+ {new Date().toLocaleTimeString()}
496
+ {:else}
497
+ —
498
+ {/if}
499
+ {:catch error}
500
+ <p style="color: red">{error.message}</p>
501
+ {/await}
502
+ </i></span>
503
+ </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
504
  </div>
505
  </div>
506
+ {/if}
507
  </div>
 
 
 
 
 
508
  </div>
509
 
510
  <style>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
511
  .panel_contents {
512
  padding: 0 20px;
513
  overflow-y: auto;
indie_label_svelte/src/SubmitReportDialog.svelte ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <script lang="ts">
2
+ import Dialog, { Title, Content, Actions } from "@smui/dialog";
3
+ import Button, { Label } from "@smui/button";
4
+ import Textfield from "@smui/textfield";
5
+ import Select, { Option } from "@smui/select";
6
+ import CircularProgress from '@smui/circular-progress';
7
+
8
+ export let open;
9
+ export let cur_user;
10
+ export let all_reports;
11
+ let email = "";
12
+ let all_sep_options = [
13
+ "Accuracy",
14
+ "Bias/Discrimination",
15
+ "Adversarial Example",
16
+ "Other",
17
+ ];
18
+ let sep_selection = "";
19
+
20
+ let promise_submit = Promise.resolve(null);
21
+ function handleSubmitReport() {
22
+ promise_submit = submitReport();
23
+ }
24
+
25
+ async function submitReport() {
26
+ let req_params = {
27
+ cur_user: cur_user,
28
+ reports: JSON.stringify(all_reports),
29
+ email: email,
30
+ sep_selection: sep_selection,
31
+ };
32
+
33
+ let params = new URLSearchParams(req_params).toString();
34
+ const response = await fetch("./submit_avid_report?" + params);
35
+ const text = await response.text();
36
+ const data = JSON.parse(text);
37
+ return data;
38
+ }
39
+
40
+ </script>
41
+
42
+ <div>
43
+ <Dialog
44
+ bind:open
45
+ aria-labelledby="simple-title"
46
+ aria-describedby="simple-content"
47
+ >
48
+ <!-- Title cannot contain leading whitespace due to mdc-typography-baseline-top() -->
49
+ <Title id="simple-title">Send All Audit Reports</Title>
50
+ <Content id="simple-content">
51
+ <!-- Description -->
52
+ <div>
53
+ <b>When you are ready to send all of your audit reports to the <a href="https://avidml.org/" target="_blank">AI Vulnerability Database</a> (AVID), please fill out the following information.</b>
54
+ Only your submitted reports will be stored in the database for further analysis. While you can submit reports anonymously, we encourage you to provide your email so that we can contact you if we have any questions.
55
+ </div>
56
+
57
+ <!-- Summary of complete reports -->
58
+ <div>
59
+ <p><b>Summary of Reports to Send</b> (Reports that include evidence and are marked as complete)</p>
60
+ <ul>
61
+ {#each all_reports as report}
62
+ {#if report["complete_status"] && (report["evidence"].length > 0)}
63
+ <li>{report["title"]}</li>
64
+ <ul>
65
+ <li>Error Type: {report["error_type"]}</li>
66
+ <li>Evidence: Includes {report["evidence"].length} example{(report["evidence"].length > 1) ? 's' : ''}</li>
67
+ <li>Summary/Suggestions: {report["text_entry"]}</li>
68
+ </ul>
69
+ {/if}
70
+ {/each}
71
+ </ul>
72
+ </div>
73
+
74
+ <!-- Form fields -->
75
+ <div>
76
+ <Select bind:value={sep_selection} label="Audit category" style="width: 90%">
77
+ {#each all_sep_options as opt}
78
+ <Option value={opt}>{opt}</Option>
79
+ {/each}
80
+ </Select>
81
+ </div>
82
+ <div>
83
+ <Textfield bind:value={email} label="(Optional) Contact email" style="width: 90%" />
84
+ </div>
85
+
86
+ <!-- Submission and status message -->
87
+ <div class="dialog_footer">
88
+ <Button on:click={handleSubmitReport} variant="outlined">
89
+ <Label>Submit Report to AVID</Label>
90
+ </Button>
91
+
92
+ <div>
93
+ <span style="color: grey"><i>
94
+ {#await promise_submit}
95
+ <CircularProgress style="height: 32px; width: 32px;" indeterminate />
96
+ {:then result}
97
+ {#if result}
98
+ Successfully sent reports! You may close this window.
99
+ {/if}
100
+ {:catch error}
101
+ <p style="color: red">{error.message}</p>
102
+ {/await}
103
+ </i></span>
104
+ </div>
105
+ </div>
106
+ </Content>
107
+ </Dialog>
108
+ </div>
109
+
110
+ <style>
111
+ :global(.mdc-dialog__surface) {
112
+ min-width: 50%;
113
+ min-height: 50%;
114
+ margin-left: 30%;
115
+ }
116
+
117
+ .dialog_footer {
118
+ padding: 20px 0px;
119
+ }
120
+ </style>
server.py CHANGED
@@ -638,34 +638,44 @@ def get_prompts_scaffold():
638
  },
639
  ]
640
 
 
 
 
 
 
 
 
 
641
  # Submit all reports to AVID
642
  # Logs the responses
643
- def submit_reports_to_AVID(reports):
644
- #Set up the connection to AVID
645
- root = environ.get('AVID_API_URL')
646
- api_key = environ.get('AVID_API_KEY')
647
  key = {"Authorization": api_key}
648
 
 
 
 
 
649
  for r in reports:
650
- new_report = utils.convert_indie_label_json_to_avid_json(r)
651
  url = root + "submit"
652
  response = requests.post(url, json=json.loads(new_report), headers=key) # The loads ensures type compliance
653
  uuid = response.json()
654
- print("AVID API response:", response, uuid)
 
 
655
 
656
  ########################################
657
  # ROUTE: /SAVE_REPORTS
658
  @app.route("/save_reports")
659
- def save_reports():
660
  cur_user = request.args.get("cur_user")
661
  reports_json = request.args.get("reports")
662
  reports = json.loads(reports_json)
663
- scaffold_method = request.args.get("scaffold_method")
664
  model = request.args.get("model")
665
 
666
- # Submit reports to AVID
667
- submit_reports_to_AVID(reports)
668
-
669
  # Save reports for current user to file
670
  reports_file = utils.get_reports_file(cur_user, model)
671
  with open(reports_file, "w", encoding ='utf8') as f:
@@ -674,7 +684,27 @@ def save_reports():
674
  results = {
675
  "status": "success",
676
  }
677
- print(results)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
678
  return json.dumps(results)
679
 
680
  ########################################
 
638
  },
639
  ]
640
 
641
+ # Filter to eligible reports: those that have been marked complete and include at least one piece of evidence.
642
+ def get_eligible_reports(reports):
643
+ eligible_reports = []
644
+ for r in reports:
645
+ if (r["complete_status"] == True) and (len(r["evidence"]) > 0):
646
+ eligible_reports.append(r)
647
+ return eligible_reports
648
+
649
  # Submit all reports to AVID
650
  # Logs the responses
651
+ def submit_reports_to_AVID(reports, cur_user, email, sep_selection, debug=DEBUG):
652
+ # Set up the connection to AVID
653
+ root = os.environ.get('AVID_API_URL')
654
+ api_key = os.environ.get('AVID_API_KEY')
655
  key = {"Authorization": api_key}
656
 
657
+ reports = get_eligible_reports(reports)
658
+ if debug:
659
+ print("Num eligible reports:", len(reports))
660
+
661
  for r in reports:
662
+ new_report = utils.convert_indie_label_json_to_avid_json(r, cur_user, email, sep_selection)
663
  url = root + "submit"
664
  response = requests.post(url, json=json.loads(new_report), headers=key) # The loads ensures type compliance
665
  uuid = response.json()
666
+ if debug:
667
+ print("Report", new_report)
668
+ print("AVID API response:", response, uuid)
669
 
670
  ########################################
671
  # ROUTE: /SAVE_REPORTS
672
  @app.route("/save_reports")
673
+ def save_reports(debug=DEBUG):
674
  cur_user = request.args.get("cur_user")
675
  reports_json = request.args.get("reports")
676
  reports = json.loads(reports_json)
 
677
  model = request.args.get("model")
678
 
 
 
 
679
  # Save reports for current user to file
680
  reports_file = utils.get_reports_file(cur_user, model)
681
  with open(reports_file, "w", encoding ='utf8') as f:
 
684
  results = {
685
  "status": "success",
686
  }
687
+ if debug:
688
+ print(results)
689
+ return json.dumps(results)
690
+
691
+ ########################################
692
+ # ROUTE: /SUBMIT_AVID_REPORT
693
+ @app.route("/submit_avid_report")
694
+ def submit_avid_report():
695
+ cur_user = request.args.get("cur_user")
696
+ email = request.args.get("email")
697
+ sep_selection = request.args.get("sep_selection")
698
+ reports_json = request.args.get("reports")
699
+
700
+ reports = json.loads(reports_json)
701
+
702
+ # Submit reports to AVID
703
+ submit_reports_to_AVID(reports, cur_user, email, sep_selection)
704
+
705
+ results = {
706
+ "status": "success",
707
+ }
708
  return json.dumps(results)
709
 
710
  ########################################