Structure outline, new toggle functionality

app.py

@@ -59,6 +59,7 @@ generation = "The strongest argument for the constitutionality of the act is tha
 gold_passage = "Unlike its original jurisdiction, the appellate jurisdiction of the Supreme Court is subject to “exceptions and regulations” prescribed by Congress, and the jurisdiction of the inferior federal courts is subject to congressional prescription. Additionally, Congress has power to regulate modes and practices of proceeding on the part of the inferior federal courts. Whether there are limitations to the exercise of these congressional powers, and what the limitations may be, are matters that have vexed scholarly and judicial interpretation over the years, inasmuch as congressional displeasure with judicial decisions has sometimes led to successful efforts to “curb” the courts and more frequently to proposed but unsuccessful curbs."
 gold_passage_generation = "The strongest argument for the constitutionality of the Congressional act is that Congress has the constitutional authority to regulate federal court jurisdiction and procedures. This includes setting limits on judicial orders. By restricting desegregation plans to only require transportation to the nearest or next nearest school, Congress is exercising its power to define and control how federal courts address such issues, ensuring judicial actions align with legislative priorities and practical concerns."
 step = 0
+mode = 0
 
 sample_dict = {
     "question": question_text,
@@ -77,7 +78,6 @@ theme = gr.themes.Soft(
     primary_hue="sky",
     secondary_hue="sky",
     neutral_hue="slate",
-    spacing_size="lg",
     font=[gr.themes.GoogleFont('Inter'), 'ui-sans-serif', 'system-ui', 'sans-serif'],
 )
 
@@ -87,7 +87,7 @@ user_id = "NO_ID"
 
 with gr.Blocks(theme = theme) as user_eval:
     # Title text introducing study
-    forward_btn = gr.Textbox("unchanged", visible = False, elem_id = "togglebutton")
+    forward_btn = gr.Textbox("unchanged", visible = False, elem_id = "togglebutton") # used for toggling windows
     gr.HTML("""
     <h1> Legal Retriever Evaluation Study </h1>
     <p> Score the passages based on the question and provided answer choices. Detailed instructions are found <a href="https://docs.google.com/document/d/1ReODJ0hlXz_M3kE2UG1cwSRVoyDLQo88OvG71Gt8lUQ/edit?usp=sharing" target="_blank">here</a>. </p> 
@@ -138,6 +138,7 @@ with gr.Blocks(theme = theme) as user_eval:
         
         def next_p(eval_1, eval_2, eval_3):
             global step
+            global mode
             step += 1
             print(eval_1 + eval_2 + eval_3)
             if step == len(passage_texts):
@@ -162,17 +163,25 @@ with gr.Blocks(theme = theme) as user_eval:
         
         def next_g(eval_helps, eval_satisfied): 
             global step 
+            global mode
             step += 1
             print(eval_helps + eval_satisfied)
             if step == 11:
-                # Step 11: guaranteed summary
+                # Step 11: guaranteed to be generation
                 return {
                     selection: gr.HTML("""
                         <h2> Autogenerated Response </h2> <br />
                         """ + generation)
                 }
             # Steps 12 and 13 are gold passage + gold passage generation IF it is applicable
-            if step > 11 and not sample_dict["top10_contains_gold_passage"]: 
+            if step > 11 and not sample_dict["top10_contains_gold_passage"]:
+                # When mode is 0 -> reset with mode = 1
+                if mode == 0:
+                    return {
+                        selection: gr.HTML("<p> Loading second set... </p>") ,
+                        forward_btn: gr.Textbox("load new data")
+                    }
+                # When mode is 1 -> display GP and GP generation, then switch
                 if step == 12: 
                     return {
                         selection: gr.HTML("""
@@ -189,12 +198,19 @@ with gr.Blocks(theme = theme) as user_eval:
                    }
                 else: 
                     return {
-                        selection: gr.Markdown("Click the button to advance to the next question."),
+                        selection: gr.Markdown("Advancing to the next question..."),
                         forward_btn: gr.Textbox("changed")
                     }
             else: 
+                # When mode is 0 -> reset with mode = 1
+                if mode == 0:
+                    return {
+                        selection: gr.HTML("<p> Loading second set... </p>") ,
+                        forward_btn: gr.Textbox("load new data")
+                    }
+                # When mode is 1 -> change question
                 return {
-                    selection: gr.Markdown("Click the button to advance to the next question."),
+                    selection: gr.Markdown("Advancing to the next question..."),
                     forward_btn: gr.Textbox("changed")                
                 }
         
@@ -224,17 +240,29 @@ with gr.Blocks(theme = theme) as user_eval:
             d.click(fn = answer, outputs = [question, evals])
     
     def toggle():
-            global step 
-            step = 0
+        global step 
+        global mode
+        step = 0
+        if mode == 0:
+            mode = 1 # update mode to 1, will restart with same Q, next set of Ps
+            print("Next set of passages for same question")
+            return {
+                scores_p: gr.Column(visible = True),
+                scores_g: gr.Column(visible = False),
+                evals: gr.Row(visible = True),
+                question: gr.Row(visible = False),
+            }
+        else:
+            mode = 0 # reset mode to 0, will restart with new Q, first set of Ps
+            print("New question")
             return {
                 scores_p: gr.Column(visible = True),
                 scores_g: gr.Column(visible = False),
                 evals: gr.Row(visible = False),
                 question: gr.Row(visible = True),
-                forward_btn: gr.HTML("unchanged")
             }
 
-    forward_btn.change(fn = toggle, inputs = None, outputs = [scores_p, scores_g, evals, question, forward_btn])
+    forward_btn.change(fn = toggle, inputs = None, outputs = [scores_p, scores_g, evals, question])
 
     with gr.Row() as login:
         with gr.Column():

structure.js

@@ -0,0 +1,35 @@
+// This is how a question will be formatted in JSON. These will be in a dictionary 
+// formatted as {question ID:question data}.
+var sample_question = {
+    "question": "What is the best explanation for this law?",
+    "answers": ["Option A", "Option B", "Option C", "Option D"],
+    "correct_answer_index": 0, 
+    "top10_e5": ["Passage 1 from E5", "Passage 2 from E5", "Passage 3..."],
+    "generation_e5": "This generation is based on the E5 passages.",
+    "top10_colbert": ["Passage 1 from ColBERT", "Passage 2 from Colbert", "..."],
+    "generation_colbert": "This generation is based on the ColBERT passages.",
+    "top10_contains_gold_passage": false, // in either case
+    "gold_passage": "This is the gold passage.",
+    "gold_passage_generation": "This generation answers the question with the gold passage."
+}
+
+// Each user response will take this form. A list of these objects will be uploaded.
+var user_response = {
+    "user_id": "[email protected]", // track who the respondent is
+    "question_id": "ID ###", // track which question they are responding to
+    "e5_scores": [[1, 2.5, 1.5, 3.5], [0, 1, 4, 1.5]], // [is a law, relevance, quality, helpfulness]
+    "colbert_scores": [[1, 3, 3, 3], [1, 5, 4, 5]], // same format as above
+}
+
+// One JSON file for each user -> check if the file exists, download if it does; otherwise, create.
+var user_data = {
+    "user_id": "[email protected]",
+    "order": ["question ID 1", "question ID 2", "...", "question ID 15"], // randomized and different for each user
+    "modes": [["e5", "colbert"], ["colbert", "e5"], ["colbert", "e5"], "..."], // randomized and different for each user
+    "current": 3, // index in "order" that the user has not yet completed (starts at 0) -> this user would log in at 3
+    "responses": [user_response, user_response] // list of user responses in the same order
+}
+// When the file exists, download it and make a LOCAL copy. Modify the local copy (e.g. increasing order 
+// and adding responses), and after each question, rewrite the original JSON object with the new data.
+// To avoid losing data in the case of a bad overwrite, each individual user_response should also be added to
+// the commit scheduler.