anonymousauthorsanonymous commited on
Commit
4f9d18b
·
1 Parent(s): 2217075

add example image

Browse files
Files changed (3) hide show
  1. .gitignore +1 -0
  2. app.py +15 -3
  3. spec_metric_result.png +0 -0
.gitignore ADDED
@@ -0,0 +1 @@
 
 
1
+ venv_unc*
app.py CHANGED
@@ -210,6 +210,16 @@ demo = gr.Blocks()
210
  with demo:
211
  input_texts = gr.Variable([])
212
  gr.Markdown("**Detect Task Specification at Inference-time.**")
 
 
 
 
 
 
 
 
 
 
213
 
214
  gr.Markdown("**Follow the numbered steps below to test one of the pre-loaded options.** Once you get the hang of it, you can load a new model and/or provide your own input texts.")
215
  gr.Markdown(f"""1) Pick a preloaded BERT-like model.
@@ -249,7 +259,7 @@ with demo:
249
  )
250
 
251
  with gr.Row():
252
- get_text_btn = gr.Button("3) Click to load input texts.)")
253
 
254
  get_text_btn.click(
255
  fn=display_input_texts,
@@ -262,7 +272,9 @@ with demo:
262
  with gr.Row():
263
  uncertain_btn = gr.Button("4) Click to get Task Specification Metric results!")
264
  gr.Markdown(
265
- "If there is an * by a sentence number, then at least one top prediction for that sentence was non-gendered.")
 
 
266
 
267
  with gr.Row():
268
  female_fig = gr.Plot(type="auto")
@@ -270,7 +282,7 @@ with demo:
270
  female_df = gr.Dataframe()
271
  with gr.Row():
272
  display_text = gr.Textbox(
273
- type="auto", label="Sample of text fed to model")
274
 
275
  uncertain_btn.click(
276
  fn=predict_gender_pronouns,
 
210
  with demo:
211
  input_texts = gr.Variable([])
212
  gr.Markdown("**Detect Task Specification at Inference-time.**")
213
+ gr.Markdown("""Well-specified tasks should have a lower specification metric value.
214
+ For example, with a close read, you can see that only Winogender schema sentence numbers (3) and (4) are well-specified:
215
+ the masked pronoun is coreferent with the `man` or `woman`, for the gendered pronoun resolution task, but the remainder are unspecfied.
216
+
217
+ In this example we have 100\% accurate detection with the specification metric near zero for only sentence (3) and (4).
218
+ <p align="center">
219
+ <img src="file/spec_metric_result.png" alt="results" width="500"/>
220
+ </p>
221
+ """)
222
+
223
 
224
  gr.Markdown("**Follow the numbered steps below to test one of the pre-loaded options.** Once you get the hang of it, you can load a new model and/or provide your own input texts.")
225
  gr.Markdown(f"""1) Pick a preloaded BERT-like model.
 
259
  )
260
 
261
  with gr.Row():
262
+ get_text_btn = gr.Button("3) Click to load input texts.")
263
 
264
  get_text_btn.click(
265
  fn=display_input_texts,
 
272
  with gr.Row():
273
  uncertain_btn = gr.Button("4) Click to get Task Specification Metric results!")
274
  gr.Markdown(
275
+ """We expect a lower specification metric for well-specified tasks.
276
+
277
+ Note: If there is an * by a sentence number, then at least one top prediction for that sentence was non-gendered.""")
278
 
279
  with gr.Row():
280
  female_fig = gr.Plot(type="auto")
 
282
  female_df = gr.Dataframe()
283
  with gr.Row():
284
  display_text = gr.Textbox(
285
+ type="text", label="Sample of text fed to model")
286
 
287
  uncertain_btn.click(
288
  fn=predict_gender_pronouns,
spec_metric_result.png ADDED