including multi-agent debate

app.py

@@ -10,6 +10,7 @@ from prompts import templates, get_examples
 from typing import Any
 from string import Template
 from qa_dreamer import get_questions
+from multi_agent_debate import MultiAgentDebate
 
 def safe_parse_json(model_answer):
     """.."""
@@ -67,7 +68,7 @@ def select_models():
     """Returns only when a valid option is selected from both dropdowns."""
 
     retriever_options = ["Choose one...", "BM25 Retriever", "Off-the-shelf Retriever", "Finetuned Retriever", "No Retriever"]
-    reasoner_options = ["Choose one...", "Claude Sonnet", "GPT-4o", "o3-mini"]
+    reasoner_options = ["Choose one...", "Claude Sonnet", "GPT-4o", "o3-mini", "Multi Agent Debate"]
     dreamer_options = ["None", "CoRE", "CoRE-Contrastive", "QA-Decomposition"]
 
     #selectboxes
@@ -275,26 +276,44 @@ def reasoner(query: str, documents: list[str], selected_dreamer: str, selected_r
                 prompt_template = Template(templates["with_evidence"])
                 prompt = prompt_template.substitute(claim=query, corpus_text=documents)
 
-        if selected_reasoner == "Claude Sonnet":
-            message += "Using Claude Sonnet to reason and verify the claim..."
-        elif selected_reasoner == "GPT-4o":
-            message += "Using GPT-4o to analyze and verify the claim in detail..."
-        elif selected_reasoner == "o3-mini":
-            message += "Using o3-mini to quickly analyze the claim..."
+        if selected_reasoner == "Multi Agent Debate":
+            multi_agent_debate = MultiAgentDebate(client=llm_client.client)
+            initial_evidence = "" if (not documents or len(documents)==0) else documents[0]
+            def mad_printer(msg, explanation):
+                print(msg)
+                ph = st.empty()
+                display_to_chat(ph, msg)
+                if explanation is not None or len(explanation) > 0:
+                    with st.expander("Agent's Argument (see more)"):
+                        st.write(str(explanation))
+                
+            decision = multi_agent_debate(claim=query,
+                                          doc=initial_evidence,
+                                          writer=mad_printer)
+            reasoning = "of the debate and discussion."
+        else:
+            if selected_reasoner == "Claude Sonnet":
+                message += "Using Claude Sonnet to reason and verify the claim..."
+            elif selected_reasoner == "GPT-4o":
+                message += "Using GPT-4o to analyze and verify the claim in detail..."
+            elif selected_reasoner == "o3-mini":
+                message += "Using o3-mini to quickly analyze the claim..."
+            elif selected_reasoner == "Multi Agent Debate":
+                message += "Multiple Agents will discuss and reason about the claim..."
 
-        print(prompt)
-        
-        llm_response = llm_client.run_inference(prompt)
+            print(prompt)
 
-        answer_dict = safe_parse_json(llm_response)
-        try:
-            decision = answer_dict.get("decision", "")
-            reasoning = answer_dict.get("reasoning", "")
-        except:
-            print(f"Error with parsing the returned {answer_dict}")
-            decision, reasoning = "", ""
+            llm_response = llm_client.run_inference(prompt)
+
+            answer_dict = safe_parse_json(llm_response)
+            try:
+                decision = answer_dict.get("decision", "")
+                reasoning = answer_dict.get("reasoning", "")
+            except:
+                print(f"Error with parsing the returned {answer_dict}")
+                decision, reasoning = "", ""
 
-        display_to_chat(placeholder, message)
+            display_to_chat(placeholder, message)
     # You could return reasoning info here.
     return reasoning, decision
 
@@ -360,6 +379,11 @@ def main():
         options["model_family"] = "OpenAI"
         options["model_name"] = "o3-mini-2025-01-31"
 
+    elif selected_reasoner == "Multi Agent Debate":
+        api_key = os.getenv("openai_key")
+        options["model_family"] = "OpenAI"
+        options["model_name"] = "gpt-4o-2024-11-20"
+
     options["API_KEY"] = api_key
 
     llm_client = LLMReasoner(options)
@@ -474,4 +498,4 @@ def main():
         st.session_state.messages.append({"role": "assistant", "content": full_response})
 
 if __name__ == '__main__':
-    main()
+    main()

multi_agent_debate.py

@@ -0,0 +1,828 @@
+# -*- coding: utf-8 -*-
+"""Untitled37.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1FbaYZ7tAm87yWo_lf87t2ynNPMR5olB-
+
+# Prep
+"""
+
+## load helper functions
+import json
+import openai
+import copy
+import random
+from openai import OpenAI
+## parsing functions
+from bs4 import BeautifulSoup
+    
+class MultiAgentDebate:
+    def __init__(self, client=None):
+        if client is not None:
+            self.client = client
+        else:
+            self.client = self.get_client()
+
+    def get_prompt_direct_eval(self, claim):
+
+        prompt = '''
+        You are given a claim in the <claim></claim> tags. Your job is to analyze a given claim and decide whether the claim is supported or not. You should also consider the provided guidelines.
+
+        <guidelines>
+        1. Evaluate the claim's plausibility based on general medical knowledge.
+        2. Consider the specificity and credibility of any numbers or percentages.
+        3. Analyze the context and scope of the claim.
+        4. Assess any potential biases or limitations.
+        </guidelines>
+
+        <claim>
+    #     %s
+        </claim>
+
+        Determine if the claim is supported or not. Provide your evaluation between <label></label> tags with values 1 (supported) or 0 (refuted) and add your explanations in <explanation></explanation> XML tags. Skip the preamble.
+        '''%(claim)
+        return prompt
+
+    def get_prompt_direct_eval_w_doc(self, doc, claim):
+
+        prompt = '''
+        You are given a claim in the <claim></claim> tags and a document as evidence in <doc></doc> tags. Your job is to analyze a given claim with respect to the given evidence and decide whether the claim is supported or not. You should also consider the provided guidelines.
+
+        <guidelines>
+        1. Evaluate the claim's plausibility based on general medical knowledge.
+        2. Consider the specificity and credibility of any numbers or percentages.
+        3. Analyze the context and scope of the claim.
+        4. Assess any potential biases or limitations.
+        </guidelines>
+
+        <doc>
+    #     %s
+        </doc>
+
+        <claim>
+    #     %s
+        </claim>
+
+        Determine if the claim is supported or not given the document as the evidence. Provide your evaluation between <label></label> tags with values 1 (supported) or 0 (refuted) and add your explanations in <explanation></explanation> XML tags. Skip the preamble.
+        '''%(doc,claim)
+        return prompt
+
+    def get_prompt_debate(self, claim, chat_history, mediator_feedback):
+        prompt = '''
+        You are given a claim in the <claim></claim> tags. Your job is to analyze a given claim and decide whether the claim is supported or not. You should also consider the provided guidelines. There are also other evaluator agents assigned the same task as you and you can also see the discussion history in <chat_history></chat_history> tags below.
+
+        <guidelines>
+        1. Evaluate the claim's plausibility based on general medical knowledge.
+        2. Consider the specificity and credibility of any numbers or percentages.
+        3. Analyze the context and scope of the claim.
+        4. Assess any potential biases or limitations.
+        </guidelines>
+
+        <claim>
+    #     %s
+        </claim>
+
+        <chat_history>
+    #     %s
+        </chat_history>
+
+        The <chat_history></chat_history> tag might be empty if this is the first round of evaluation. You can see your previous responses as well as other agents responses. Continue the discussion with other evaluator agents, talk to them and state why you agree/disagree with each other bringing as many arguments as you can. %s Determine if the claim is supported or not. Provide your evaluation between <label></label> tags with values 1 (supported) or 0 (refuted) and add your arguments in <argument ></argument> XML tags. Skip the preamble.
+        '''%(claim,chat_history,mediator_feedback)
+        return prompt
+
+    def get_adjudicator_prompt(self, claim, chat_history):
+        prompt = '''
+        You are given a claim in the <claim></claim> tags and multiple judgments from evaluator agents. You go over the discussion between the agents and their arguments shown in between <chat_history></chat_history> tags. Your job is to analyze a given claim and decide whether the claim is supported or not. You should also consider the provided guidelines.
+
+        <guidelines>
+        1. Evaluate the claim's plausibility based on general medical knowledge.
+        2. Consider the specificity and credibility of any numbers or percentages.
+        3. Analyze the context and scope of the claim.
+        4. Assess any potential biases or limitations.
+        </guidelines>
+
+        <claim>
+    #     %s
+        </claim>
+
+        <chat_history>
+    #     %s
+        </chat_history>
+
+        Go over the agents responses, summarize them by saying who agrees/disagrees. Then looking at the agents responses, how well they are associated with the guidelines and finally your own judgement, determine if the claim is supported or not. Provide your evaluation between <label></label> tags with values 1 (supported) or 0 (refuted) and add your arguments in <argument ></argument> XML tags. Skip the preamble.
+        '''%(claim,chat_history)
+        #Go over the agents responses, summarize them by saying who agrees/disagrees and make sure the arguments correctly used the provided guidelines. Then based on the correctness of agents responses and your own judegment of the summary using the provided guidelines, determine if the sentence is factually consistent with the document. A summary is factually inconsistent if there is a correct argument describing an error or discrepancy in the summary. Provide your evaluation using a JSON format with keys as "label" with values 1 (consistent) or 0 (inconsistent) and "explanation" and put your response between <response></response> tags. Skip the preamble.
+        return prompt
+
+    def get_prompt_debate_w_doc(self, doc, claim, chat_history, mediator_feedback):
+        prompt = '''
+        You are given a claim in the <claim></claim> tags and a document as evidence in <doc></doc> tags. Your job is to analyze a given claim and decide whether the claim is supported or not with respect to the given evidence. You should also consider the provided guidelines. There are also other evaluator agents assigned the same task as you and you can also see the discussion history in <chat_history></chat_history> tags below.
+
+        <guidelines>
+        1. Evaluate the claim's plausibility based on general medical knowledge.
+        2. Consider the specificity and credibility of any numbers or percentages.
+        3. Analyze the context and scope of the claim.
+        4. Assess any potential biases or limitations.
+        </guidelines>
+
+        <doc>
+    #     %s
+        </doc>
+
+        <claim>
+    #     %s
+        </claim>
+
+        <chat_history>
+    #     %s
+        </chat_history>
+
+        The <chat_history></chat_history> tag might be empty if this is the first round of evaluation. You can see your previous responses as well as other agents responses. Continue the discussion with other evaluator agents, talk to them and state why you agree/disagree with each other bringing as many arguments as you can. %s Determine if the claim is supported or not. Provide your evaluation between <label></label> tags with values 1 (supported) or 0 (refuted) and add your arguments in <argument ></argument> XML tags. Skip the preamble.
+        '''%(doc, claim,chat_history,mediator_feedback)
+        return prompt
+
+    def get_adjudicator_prompt_w_doc(self, doc, claim, chat_history):
+        prompt = '''
+        You are given a claim in the <claim></claim> tags, a document as evidence in <doc></doc> tags and multiple judgments from evaluator agents. You go over the discussion between the agents and their arguments shown in between <chat_history></chat_history> tags. Your job is to analyze a given claim and decide whether the claim is supported or not with respect to the given evidence. You should also consider the provided guidelines.
+
+        <guidelines>
+        1. Evaluate the claim's plausibility based on general medical knowledge.
+        2. Consider the specificity and credibility of any numbers or percentages.
+        3. Analyze the context and scope of the claim.
+        4. Assess any potential biases or limitations.
+        </guidelines>
+
+        <doc>
+    #     %s
+        </doc>
+
+        <claim>
+    #     %s
+        </claim>
+
+        <chat_history>
+    #     %s
+        </chat_history>
+
+        Go over the agents responses, summarize them by saying who agrees/disagrees. Then looking at the agents responses, how well they are associated with the guidelines and finally your own judgement, determine if the claim is supported or not. Provide your evaluation between <label></label> tags with values 1 (supported) or 0 (refuted) and add your arguments in <argument ></argument> XML tags. Skip the preamble.
+        '''%(doc, claim,chat_history)
+        #Go over the agents responses, summarize them by saying who agrees/disagrees and make sure the arguments correctly used the provided guidelines. Then based on the correctness of agents responses and your own judegment of the summary using the provided guidelines, determine if the sentence is factually consistent with the document. A summary is factually inconsistent if there is a correct argument describing an error or discrepancy in the summary. Provide your evaluation using a JSON format with keys as "label" with values 1 (consistent) or 0 (inconsistent) and "explanation" and put your response between <response></response> tags. Skip the preamble.
+        return prompt
+
+    def get_prompt_direct_w_causal_sub_claims(self, claim):
+
+        prompt = '''
+        You are given a claim in the <claim></claim> tags. Your job is to first break the claim into causal sub-claims and then analyze each sub-claim and decide whether the whole claim is supported or not.
+        A causal sub-claim represents a causal relation between two entities from the claim. This is not simply a statement but rather an explicit causal relation in which one entity (or its change) can have an effect on the other entity.
+        If all sub-claims are supported, then the claim is also supported. You should also consider the provided guidelines.
+
+        <guidelines>
+        1. Evaluate the claim's plausibility based on general medical knowledge.
+        2. Consider the specificity and credibility of any numbers or percentages.
+        3. Analyze the context and scope of the claim.
+        4. Assess any potential biases or limitations.
+        </guidelines>
+
+        <claim>
+        40mg/day dosage of folic acid and 2mg/day dosage of vitamin B12 does not affect chronic kidney disease (CKD) progression.
+        </claim>
+
+        Break the claim into any possible number of causal sub-claims with explicit causal relations and place them in <sub-claims></sub-claims> tags. Determine if the claim is supported or not given the document as the evidence by analyzing its causal sub-claims. Provide your evaluation between <label></label> tags with values 1 (supported) or 0 (refuted) and add your explanations in <argument></argument> XML tags. Skip the preamble.
+
+        <sub-claims>
+        40mg/day dosage of folic acid causes chronic kidney disease (CKD) progression changes.
+        2mg/day dosage of vitamin B12 causes chronic kidney disease (CKD) progression changes.
+        </sub-claims>
+
+        <label>
+        1
+        </label>
+
+        <argument>
+        Yes. There is a study that indicates that treatment with high doses of folic acid (40 mg/day) did not improve survival or reduce the incidence of vascular disease in patients with advanced chronic kidney disease or end-stage renal disease. The study found no significant effect on mortality or secondary outcomes related to CKD progression, such as myocardial infarction, stroke, amputation, time to dialysis, or thrombosis in hemodialysis patients. Therefore, the claim that these dosages do not affect CKD progression is supported by the study's findings. It is also mentioned that patients who received vitamin B12 (2 mg/day), also did not show any significant effects on outcomes.
+        </argument>
+
+        You are given a claim in the <claim></claim> tags. Your job is to first break the claim into causal sub-claims and then analyze each sub-claim and decide whether the whole claim is supported or not.
+        A causal sub-claim represents a causal relation between two entities from the claim. This is not simply a statement but rather an explicit causal relation in which one entity (or its change) can have an effect on the other entity.
+        If all sub-claims are supported, then the claim is also supported. You should also consider the provided guidelines.
+
+        <guidelines>
+        1. Evaluate the claim's plausibility based on general medical knowledge.
+        2. Consider the specificity and credibility of any numbers or percentages.
+        3. Analyze the context and scope of the claim.
+        4. Assess any potential biases or limitations.
+        </guidelines>
+
+        <claim>
+    #     %s
+        </claim>
+
+        Break the claim into causal sub-claims and place them in <sub-claims></sub-claims> tags. Determine if the claim is supported or not given the document as the evidence by analyzing its causal sub-claims. Provide your evaluation between <label></label> tags with values 1 (supported) or 0 (refuted) and add your explanations in <argument></argument> XML tags. Skip the preamble.
+        '''%(claim)
+        return prompt
+
+    def get_prompt_direct_w_doc_w_causal_sub_claims(self, doc, claim):
+
+        prompt = '''
+        You are given a claim in the <claim></claim> tags and a document as evidence in <doc></doc> tags. Your job is to first break the claim into causal sub-claims and then analyze each sub-claim with respect to the given evidence and decide whether the whole claim is supported or not.
+        A causal sub-claim represents a causal relation between two entities from the claim. This is not simply a statement but rather an explicit causal relation in which one entity (or its change) can have an effect on the other entity.
+        If all sub-claims are supported, then the claim is also supported. You should also consider the provided guidelines.
+
+        <guidelines>
+        1. Evaluate the claim's plausibility based on general medical knowledge.
+        2. Consider the specificity and credibility of any numbers or percentages.
+        3. Analyze the context and scope of the claim.
+        4. Assess any potential biases or limitations.
+        </guidelines>
+
+        <doc>
+        High plasma homocysteine levels are a risk factor for mortality and vascular disease in observational studies of patients with chronic kidney disease.", "Folic acid and B vitamins decrease homocysteine levels in this population but whether they lower mortality is unknown. \n", "OBJECTIVE To determine whether high doses of folic acid and B vitamins administered daily reduce mortality in patients with chronic kidney disease. \n", "DESIGN, SETTING, AND PARTICIPANTS Double-blind randomized controlled trial (2001-2006) in 36 US Department of Veterans Affairs medical centers.", "Median follow-up was 3.2 years for 2056 participants aged 21 years or older with advanced chronic kidney disease (estimated creatinine clearance < or =30 mL/min) (n = 1305) or end-stage renal disease (n = 751) and high homocysteine levels (> or = 15 micromol/L). \n", "INTERVENTION Participants received a daily capsule containing 40 mg of folic acid, 100 mg of pyridoxine hydrochloride (vitamin B6), and 2 mg of cyanocobalamin (vitamin B12) or a placebo. \n", "MAIN OUTCOME MEASURES The primary outcome was all-cause mortality.", "Secondary outcomes included myocardial infarction (MI), stroke, amputation of all or part of a lower extremity, a composite of these 3 plus all-cause mortality, time to initiation of dialysis, and time to thrombosis of arteriovenous access in hemodialysis patients. \n", "RESULTS Mean baseline homocysteine level was 24.0 micromol/L in the vitamin group and 24.2 micromol/L in the placebo group.", "It was lowered 6.3 micromol/L (25.8%%, P < .001) in the vitamin group and 0.4 micromol/L (1.7%%, P = .14) in the placebo group at 3 months, but there was no significant effect on mortality (448 vitamin group deaths vs 436 placebo group deaths) (hazard ratio [HR], 1.04, 95%% CI, 0.91-1.18).", "No significant effects were demonstrated for secondary outcomes or adverse events: there were 129 MIs in the vitamin group vs 150 for placebo (HR, 0.86, 95%% CI, 0.67-1.08), 37 strokes in the vitamin group vs 41 for placebo (HR, 0.90, 95%% CI, 0.58-1.40), and 60 amputations in the vitamin group vs 53 for placebo (HR, 1.14, 95%% CI, 0.79-1.64).", "In addition, the composite of MI, stroke, and amputations plus mortality (P = .85), time to dialysis (P = .38), and time to thrombosis in hemodialysis patients (P = .97) did not differ between the vitamin and placebo groups. \n", "CONCLUSION Treatment with high doses of folic acid and B vitamins did not improve survival or reduce the incidence of vascular disease in patients with advanced chronic kidney disease or end-stage renal disease. \n", "TRIAL REGISTRATION clinicaltrials.gov Identifier: NCT00032435."
+        </doc>
+
+        <claim>
+        40mg/day dosage of folic acid and 2mg/day dosage of vitamin B12 does not affect chronic kidney disease (CKD) progression.
+        </claim>
+
+        Break the claim into any possible number of causal sub-claims with explicit causal relations and place them in <sub-claims></sub-claims> tags. Determine if the claim is supported or not given the document as the evidence by analyzing its causal sub-claims. Provide your evaluation between <label></label> tags with values 1 (supported) or 0 (refuted) and add your explanations in <argument></argument> XML tags. Skip the preamble.
+
+        <sub-claims>
+        40mg/day dosage of folic acid causes chronic kidney disease (CKD) progression changes.
+        2mg/day dosage of vitamin B12 causes chronic kidney disease (CKD) progression changes.
+        </sub-claims>
+
+        <label>
+        1
+        </label>
+
+        <argument>
+        Yes. The information provided indicates that treatment with high doses of folic acid (40 mg/day) did not improve survival or reduce the incidence of vascular disease in patients with advanced chronic kidney disease or end-stage renal disease. The study found no significant effect on mortality or secondary outcomes related to CKD progression, such as myocardial infarction, stroke, amputation, time to dialysis, or thrombosis in hemodialysis patients. Therefore, the claim that these dosages do not affect CKD progression is supported by the study's findings. It is also mentioned that patients who received vitamin B12 (2 mg/day), also did not show any significant effects on outcomes.
+        </argument>
+
+        You are given a claim in the <claim></claim> tags and a document as evidence in <doc></doc> tags. Your job is to first break the claim into causal sub-claims and then analyze each sub-claim with respect to the given evidence and decide whether the whole claim is supported or not.
+        A causal sub-claim represents a causal relation between two entities from the claim. This is not simply a statement but rather an explicit causal relation in which one entity (or its change) can have an effect on the other entity.
+        If all sub-claims are supported, then the claim is also supported. You should also consider the provided guidelines.
+
+        <guidelines>
+        1. Evaluate the claim's plausibility based on general medical knowledge.
+        2. Consider the specificity and credibility of any numbers or percentages.
+        3. Analyze the context and scope of the claim.
+        4. Assess any potential biases or limitations.
+        </guidelines>
+
+        <doc>
+    #     %s
+        </doc>
+
+        <claim>
+    #     %s
+        </claim>
+
+        Break the claim into causal sub-claims and place them in <sub-claims></sub-claims> tags. Determine if the claim is supported or not given the document as the evidence by analyzing its causal sub-claims. Provide your evaluation between <label></label> tags with values 1 (supported) or 0 (refuted) and add your explanations in <argument></argument> XML tags. Skip the preamble.
+        '''%(doc,claim)
+        return prompt
+
+    def parse_output_response(self, response):
+        soup = BeautifulSoup(response, 'html.parser')
+        explanation_list = soup.find_all("explanation")
+        explanation_text = ""
+        for exp in explanation_list:
+            if exp.string != None:
+                explanation_text += exp.string + ' '
+            else:
+                explanation_text = response
+        explanation_text = ' '.join(explanation_text.split())
+        if len(soup.find_all("label")) > 0:
+            labels = soup.find_all("label")[-1].string.strip()
+        else:
+            labels = "Unknown"
+        return labels, explanation_text
+
+    def parse_output_response_w_category(self, response):
+        soup = BeautifulSoup(response, 'html.parser')
+        explanation_list = soup.find_all("explanation")
+        explanation_text = ""
+        for exp in explanation_list:
+            if exp.string != None:
+                explanation_text += exp.string + ' '
+            else:
+                explanation_text = response
+        explanation_text = ' '.join(explanation_text.split())
+
+        category_list = soup.find_all("category")
+        category_text = ""
+        for exp in category_list:
+            if exp.string != None:
+                category_text += exp.string + ' '
+            else:
+                category_text = ""
+        category_text = ' '.join(category_text.split())
+
+        if len(soup.find_all("label")) > 0:
+            labels = soup.find_all("label")[-1].string.strip()
+        else:
+            labels = "Unknown"
+
+        return labels, category_text, explanation_text
+
+    def parse_output_w_chat_label(self, response):
+        soup = BeautifulSoup(response, 'html.parser')
+        argument_list = soup.find_all("argument")
+        argument_text = ""
+        for argument in argument_list:
+            if argument.string != None:
+                argument_text += argument.string + ' '
+            else:
+                argument_text = response
+        argument_text = ' '.join(argument_text.split())
+        if len(soup.find_all("label")) > 0:
+            guidelines = soup.find_all("label")[0].string.strip()
+        else:
+            guidelines = "Unknown"
+
+        return argument_text, guidelines
+
+    def parse_output_response_w_causal_subclaims(self, response):
+        soup = BeautifulSoup(response, 'html.parser')
+        argument_list = soup.find_all("argument")
+        argument_text = ""
+        for argument in argument_list:
+            if argument.string != None:
+                argument_text += argument.string + ' '
+            else:
+                argument_text = response
+
+        argument_text = ' '.join(argument_text.split())
+        if len(soup.find_all("label")) > 0:
+            label = soup.find_all("label")[0].string.strip()
+        else:
+            label = "Unknown"
+
+        sub_claims_text = ""
+        if len(soup.find_all("sub-claims")) > 0:
+            sub_claims_list = soup.find_all("sub-claims")
+            for claim in sub_claims_list:
+                    if claim.string != None:
+                        sub_claims_text += claim.string + '\n'
+
+        return label, argument_text, sub_claims_text
+
+    """# OpenAI Prep"""
+
+    def get_client(self):
+      self.client = OpenAI(api_key="",
+                      organization="")
+      return self.client
+
+    #client = get_client()
+    def parse_chatgpt_api_response(self, response):
+        choices = response.choices
+        # choices = response["choices"]
+        main_response_message_list = []
+        if len(choices) > 1:
+            for choice in choices:
+                main_response = choice.message
+                # main_response_message, main_response_role = main_response["content"], main_response["role"]
+                main_response_message, main_response_role = main_response.content, main_response.role
+                main_response_message_list.append(main_response_message)
+            return main_response_message_list, response
+
+        else:
+            main_response = choices[0].message
+            # main_response_message, main_response_role = main_response["content"], main_response["role"]
+            main_response_message, main_response_role = main_response.content, main_response.role
+            return main_response_message, response
+
+    def make_openai_api_call(self, prompt, model_name, temperature):
+        if 'gpt-3' in model_name or'gpt-4' in model_name:
+            # openai.ChatCompletion.create
+            response = self.client.chat.completions.create(
+                model=model_name,
+                messages=[{'role': 'user', 'content': prompt}],
+                temperature=temperature,
+                max_tokens=4096,
+                top_p=1.0,
+                frequency_penalty=0.0,
+                presence_penalty=0.0,
+                n=1,
+            )
+            return self.parse_chatgpt_api_response(response)
+
+    def make_openai_api_call_o3_mini(self, prompt, model_name, temperature):
+        response = self.client.chat.completions.create(
+        model=model_name,
+        messages=[{'role': 'user', 'content': prompt}],
+        response_format={
+            "type": "text"
+        },
+        reasoning_effort="medium"
+        )
+        return self.parse_chatgpt_api_response(response)
+
+    def read_file(self, file_path):
+        all_data = []
+        with open(file_path, 'r') as input_file:
+            for line in input_file:
+                line = line.strip()
+                data = json.loads(line)
+                all_data.append(data)
+        return all_data
+
+    def safe_print(self, x, *args):
+        print(x)
+
+    def __call__(self, doc, claim, initialization=True, model_name='gpt-4o-mini',
+                  writer=safe_print):
+        # number of simultaneous debates for evaluation
+        num_debates = 1
+        eval_repeat_max = 0
+
+        ## initilaize a dictionary to save the outputs of each separate debate
+        debates_dict = dict.fromkeys([0],None)
+        overall_ambiguity = False
+        initialization = initialization
+
+        ## keep starting debates until you reach the max numer of debates
+        while eval_repeat_max != num_debates:
+            ambiguous = False
+            results = {}
+            doc = doc
+            sent = claim
+
+            ## intial stance assignment. We use the follwoing list of utterances as the first reponse of each agent and then use
+            ## this as the chat history to start the debate. The default value is 4. You can change the number of agents by adding
+            ## more utterances
+
+            if initialization:
+                agents_responses = ["The claim is not refuted by evidence.", "The claim is refuted by evidence.", "The claim is not refuted by evidence.", "The claim is refuted by evidence."]
+            else:
+                agents_responses = ["","","",""]
+
+            updated_responses = []
+
+            ## to keep track of previous responses of agents and provide them in each round
+            message_board = ['','','','']
+
+            ## intialize a label list to keep track of agents judgements
+            label_list = [[1],[0],[1],[0]]
+            all_chats = []
+
+            ## number of rounds of debates
+            turns = 3
+
+            mediator_feedback = ""
+            ## first round of random assessment not included in the history.
+            round_counter = 0
+            if initialization:
+                print("ROUND %s: (This is the initialization round where agents are assigned initial stance as their beliefs.)\n"%str(round_counter+1))
+                for n in range(len(agents_responses)):
+                    print("Agent %s: "%str(n+1) + agents_responses[n] + "\n")
+                print("----------------------------------------------------")
+                round_counter += 1
+            print("ROUND %s:\n"%str(round_counter+1))
+            for n in range(len(agents_responses)):
+                chat_history = ""
+                chat_history_prompt = ''
+                chat_history_prompt +=  message_board[n] + "You (Agent %s): "%str(n+1) + agents_responses[n] + "\n"
+                chat_history += "You (Agent %s): "%str(n+1) + agents_responses[n] + "\n"
+                other_agents_response = ""
+                for nn in range(len(agents_responses)):
+                    if nn != n:
+                        other_agents_response += "Agent %s: "%str(nn+1) + agents_responses[nn] + "\n"
+                        chat_history += "Agent %s: "%str(nn+1) + agents_responses[nn] + "\n"
+
+                message_board[n] += chat_history
+                chat_history_prompt += other_agents_response
+
+                ## For experiments wo initial stance uncomment the following line to clear the chat history
+                if not initialization:
+                    chat_history_prompt = ""
+
+                ## the parameters to prompt module include the document, the claim sentence, previous chat history and mediator feedback
+                ## that you can use to modify the goals of agents
+                if doc != "":
+                    prompt = self.get_prompt_debate_w_doc(doc, sent, chat_history_prompt, mediator_feedback)
+                else:
+                    prompt = self.get_prompt_debate(sent, chat_history_prompt, mediator_feedback)
+                argument = ""
+                rep_ctr = 0
+                label = -1
+                label_val = -1
+
+                ## to make sure we have enough initial diversity in responses, we repeat the following such that if the immediate
+                ## response is different from the assigned stance, the agent is asked to repeat its generation. The rep_ctr is used
+                ## to repaet 2 times before moving on to the next stage
+                while label!="Unknown" and label_val != label_list[n][0] and rep_ctr != 1:
+                    llm_response, _ = self.make_openai_api_call(prompt, model_name, 1)
+                    argument, label = self.parse_output_w_chat_label(llm_response)
+                    strlabel = "Support" if label == 1 else "Refute"
+                    writer("Agent %s's Assessment:\n"%str(n+1) + '%s. \n'%strlabel, 'Explanation: %s'%argument + "\n")
+                    print("***************")
+                    rep_ctr += 1
+
+                    ## the generated label might not be in correct format so we use the following to make sure the label format is correct
+                    if label != "Unknown":
+                        if len(label.split()) != 0 and ',' not in label.split()[0]:
+                            label_val = float(label.split()[0])
+                        elif len(label.split()) == 0 or ',' in label.split()[0]:
+                            if len(label.split(',')) != 0:
+                                label_val = float(label.split(',')[0])
+                            else:
+                                label_val = float(label)
+
+                        if label_val >= 0.5:
+                            label_val = 1
+                        else:
+                            label_val = 0
+
+                if label != "Unknown":
+                    if len(label.split()) != 0 and ',' not in label.split()[0]:
+                        label_val = float(label.split()[0])
+                    elif len(label.split()) == 0 or ',' in label.split()[0]:
+                        if len(label.split(',')) != 0:
+                            label_val = float(label.split(',')[0])
+                        else:
+                            label_val = float(label)
+
+                    if label_val >= 0.5:
+                        label_list[n].append(1)
+                    else:
+                        label_list[n].append(0)
+                else:
+                    label_list[n].append(label_list[n][-1])
+                argument = argument.strip()
+
+                updated_responses.append(argument)
+            agents_responses = copy.deepcopy(updated_responses)
+
+            ## Once the first round is generated, we start the debate among agents
+            message_board = ['','','','']
+            for ag, ag_resp in enumerate(agents_responses):
+                all_chats.append("Agent %s:\n"%str(ag+1) + ag_resp)
+
+            mediator_feedback = ""
+
+            ## The debate is continued for "turns" time.
+            for cnt in range(turns):
+                if len(set([lbl_list[-1] for lbl_list in label_list])) == 1:
+                    break
+                print("----------------------------------------------------")
+                round_counter += 1
+                print("ROUND %s:\n"%str(round_counter+1))
+                updated_responses = []
+                for n in range(len(agents_responses)):
+                    chat_history = ""
+                    chat_history_prompt = ''
+                    chat_history_prompt +=  message_board[n] + "You (Agent %s): "%str(n+1) + agents_responses[n] + "\n"
+                    chat_history += "You (Agent %s): "%str(n+1) + agents_responses[n] + "\n"
+                    other_agents_response = ""
+                    for nn in range(len(agents_responses)):
+                        if nn != n:
+                            other_agents_response += "Agent %s: "%str(nn+1) + agents_responses[nn] + "\n"
+                            chat_history += "Agent %s: "%str(nn+1) + agents_responses[nn] + "\n"
+
+                    message_board[n] += chat_history
+                    chat_history_prompt += other_agents_response
+
+                    ## to shuffle the order of chat history to remove any biases caused by order of chats
+                    new_chat_history_list = []
+                    chat_history_prompt_list = chat_history_prompt.split('\n')
+                    chat_history_prompt_list = [chat_hist for chat_hist in chat_history_prompt_list if chat_hist != ""]
+                    for pq in range(0,len(chat_history_prompt_list),len(agents_responses)):
+                        shuffled_list = chat_history_prompt_list[pq:pq+len(agents_responses)]
+                        random.shuffle(shuffled_list)
+                        new_chat_history_list += shuffled_list
+                    chat_history_prompt = '\n'.join(new_chat_history_list)
+
+                    ## you can add any type of feedback here and add them to prompt to improve the debate consensus
+                    ## we do it after the first round
+                    # if cnt >= 1:
+                    #     mediator_feedback = " Look back at the guidelines and how you have used them. Make sure all guidelines (and not only a subset of them) are satisfied in your assessment. Change your stance if you have made an error or if the other agents are more convincing."
+                    mediator_feedback = ""
+
+                    if doc != "":
+                        prompt = self.get_prompt_debate_w_doc(doc, sent, chat_history_prompt, mediator_feedback)
+                    else:
+                        prompt = self.get_prompt_debate(sent, chat_history_prompt, mediator_feedback)
+                    llm_response, _ = self.make_openai_api_call(prompt, model_name, 1)
+                    # print(llm_response)
+                    # print("***************")
+                    argument, label = self.parse_output_w_chat_label(llm_response)
+                    strlabel = "Support" if label == 1 else "Refute"
+                    writer("Agent %s's Assessment: \n"%str(n+1) + '%s. \n'%strlabel, 'Explanation: %s'%argument + "\n")
+                    print("***************")
+                    if label != "Unknown":
+                        if len(label.split()) != 0 and ',' not in label.split()[0]:
+                            label_val = float(label.split()[0])
+                        elif len(label.split()) == 0 or ',' in label.split()[0]:
+                            if len(label.split(',')) != 0:
+                                label_val = float(label.split(',')[0])
+                            else:
+                                label_val = float(label)
+
+                        if label_val >= 0.5:
+                            label_list[n].append(1)
+                        else:
+                            label_list[n].append(0)
+                    else:
+                        label_list[n].append(label_list[n][-1])
+                    argument = argument.strip()
+
+                    updated_responses.append(argument)
+                    all_chats.append('Agent %s:\n'%str(n+1) + argument)
+                agents_responses = copy.deepcopy(updated_responses)
+                if len(set([lbl_list[-1] for lbl_list in label_list])) == 1:
+                    break
+
+            #print(label_list)
+            label_list_text = [["Supported" if item == 1 else "Refuted" for item in lbl] for lbl in label_list]
+            print('----------------------------------------------------')
+            for lbl in range(len(label_list_text)):
+                print("Agent %s trajectory:\n%s\n"%(str(lbl+1), label_list_text[lbl]))
+
+
+            pn_list = [lbl[-1] for lbl in label_list]
+            debate_arguments = copy.deepcopy(all_chats[-len(agents_responses):])
+
+            ## we record the outputs of the debate in a dictionary that was previously initialized.
+            ## the "change" key keeps track of the number of agents who changes their stance during debate.
+            ## this can be used to identify the ambiguous cases directly.
+            if pn_list.count(0) == pn_list.count(1):
+                debates_dict[eval_repeat_max] = {'change': 0, 'label': -1,'arguments': debate_arguments,'labels': label_list}
+
+                all_chats_dict = {}
+                for n_agents in range(len(debate_arguments)):
+                    all_chats_dict['Agent %s:'%str(n_agents+1)] = ""
+
+                for cht_counter, cht in enumerate(debate_arguments):
+                    all_chats_dict['Agent %s:'%str(cht_counter+1)] += ' '.join(cht.split('\n')[1:]) + ' '
+
+                ## if there is not a winner label, we use adjudicators to decide on the final label.
+                ## you can use multiple adjudicators if you want to do majority voting among them.
+                adjudicator_input = [str(item) + ' ' + all_chats_dict[item] for item in all_chats_dict]
+                if doc != "":
+                    adjudicator_prompt = self.get_adjudicator_prompt_w_doc(doc, sent, '\n'.join(adjudicator_input))
+                else:
+                    adjudicator_prompt = self.get_adjudicator_prompt(sent, '\n'.join(adjudicator_input))
+                rep_counter = 0
+                adjudicator_label_list = []
+                label = ""
+                explanation_list = []
+                for i in range(1):
+                    while label == "" and rep_counter != 5:
+                        adjudicator_response, _ = self.make_openai_api_call(adjudicator_prompt, model_name, 1.0)
+                        label ,  explanation  = self.parse_output_response(adjudicator_response)
+                        explanation_list.append(explanation)
+                        writer(label, explanation)
+                        print('********')
+                        if label != "Unknown":
+                            if len(label.split()) != 0 and ',' not in label.split()[0]:
+                                label_val = float(label.split()[0])
+                            elif len(label.split()) == 0 or ',' in label.split()[0]:
+                                if len(label.split(',')) != 0:
+                                    label_val = float(label.split(',')[0])
+                                else:
+                                    label_val = float(label)
+                            if label_val >= 0.5:
+                                label = 1
+                            else:
+                                label = 0
+                        else:
+                            label = -1
+                        rep_counter += 1
+                    adjudicator_label_list.append(label)
+                    label = ""
+
+                if adjudicator_label_list.count(1) >= adjudicator_label_list.count(0):
+                    label = 1
+                else:
+                    label = 0
+                debates_dict[eval_repeat_max]['label'] = label
+
+            ## if there is a winner label, we return the winner as the final label of the claim
+            elif pn_list.count(0) != pn_list.count(1):
+                if pn_list.count(1) >= pn_list.count(0):
+                    label = 1
+                else:
+                    label = 0
+
+                if len(set(pn_list)) == 1:
+                    change = len(agents_responses)//2
+                else:
+                    change = len(agents_responses)//2 - 1
+                debates_dict[eval_repeat_max] = {'change': change, 'label': label,'arguments': debate_arguments,'labels': label_list}
+                explanation_list = debate_arguments
+
+            eval_repeat_max += 1
+
+        all_label_lists = [debates_dict[item]['labels'] for item in debates_dict]
+
+        ## majority vote out of debate rounds. There is a winner for each debate and then the final winner is the one with the most votes
+        debates_majority_vote_list = [debates_dict[item]['label'] for item in debates_dict]
+        print(debates_majority_vote_list)
+        if debates_majority_vote_list.count(1) == num_debates or debates_majority_vote_list.count(0) == num_debates:
+            debate_ambiguity = False
+        else:
+            debate_ambiguity = True
+
+        if debates_majority_vote_list.count(1)> debates_majority_vote_list.count(0):
+            debates_majority_vote = 1
+        elif debates_majority_vote_list.count(1) < debates_majority_vote_list.count(0):
+            debates_majority_vote = 0
+        print(debates_majority_vote)
+
+        changes_in_debates_list = [debates_dict[item]['change'] for item in debates_dict]
+        if changes_in_debates_list.count(0) == num_debates:
+            ambiguous = "Full"
+        elif changes_in_debates_list.count(0) == 0:
+            ambiguous = "None"
+        else:
+            ambiguous = "Partial"
+
+        # if changes_in_debates_list.count(0) != num_debates:
+        overall_majority_list = []
+        for label_list in all_label_lists:
+            change = 0
+            pn_list = []
+            for lbl in label_list:
+                if lbl[0] != lbl[-1]:
+                    change += 1
+                pn_list.append(lbl[-1])
+            overall_majority_list += pn_list
+
+        ## majority vote over all individual agents regardless of which debate they belong to
+        if overall_majority_list.count(1)> overall_majority_list.count(0):
+            overall_majority_vote = 1
+        elif overall_majority_list.count(1) < overall_majority_list.count(0):
+            overall_majority_vote = 0
+        else:
+            overall_ambiguity = True
+
+        ## if there is a winner among the agents responses, we report the majority vote
+        if changes_in_debates_list.count(0) != num_debates and overall_ambiguity == False:
+            label = overall_majority_vote
+            explanation_list = [debates_dict[item]['arguments'] for item in debates_dict]
+            adjudicator_list = []
+            all_arguments = [debates_dict[item]['arguments'] for item in debates_dict]
+
+        ## if there is NOT a winner among agents responses, we use adjudicators to make the final call
+        elif changes_in_debates_list.count(0) == num_debates or overall_ambiguity == True:
+            all_arguments = [debates_dict[item]['arguments'] for item in debates_dict]
+            all_arguments = [x for xs in all_arguments for x in xs]
+            all_chats_dict = {}
+            for n_agents in range(len(all_arguments)):
+                all_chats_dict['Agent %s:'%str(n_agents+1)] = ""
+
+            for cht_counter, cht in enumerate(all_arguments):
+                all_chats_dict['Agent %s:'%str(cht_counter+1)] += ' '.join(cht.split('\n')[1:]) + ' '
+
+            adjudicator_input = [str(item) + ' ' + all_chats_dict[item] for item in all_chats_dict]
+
+            label_list = []
+            label = ""
+            explanation_list = []
+            for rep in range(3):
+                random.shuffle(adjudicator_input)
+                if doc != "":
+                    adjudicator_prompt = self.get_adjudicator_prompt_w_doc(doc, sent, '\n'.join(adjudicator_input))
+                else:
+                    adjudicator_prompt = get_adjudicator_prompt(sent, '\n'.join(adjudicator_input))
+                rep_counter = 0
+                while label == "" and rep_counter != 5:
+                    adjudicator_response, _ = self.make_openai_api_call(adjudicator_prompt, model_name, 1.0)
+                    label ,  explanation  = self.parse_output_response(adjudicator_response)
+                    explanation_list.append(explanation)
+                    writer(label, explanation)
+                    print('********')
+                    if label != "Unknown":
+                        if len(label.split()) != 0 and ',' not in label.split()[0]:
+                            label_val = float(label.split()[0])
+                        elif len(label.split()) == 0 or ',' in label.split()[0]:
+                            if len(label.split(',')) != 0:
+                                label_val = float(label.split(',')[0])
+                            else:
+                                label_val = float(label)
+                        if label_val >= 0.5:
+                            label = 1
+                        else:
+                            label = 0
+                    else:
+                        label = -1
+                    rep_counter += 1
+                label_list.append(label)
+                label = ""
+
+            print(label_list)
+            results['adjudicators'] = label_list
+            results['adjudicators_agree'] = len(set(label_list)) == 1
+            if label_list.count(1) >= label_list.count(0):
+                label = 1
+            else:
+                label = 0
+
+            overall_majority_vote = label
+            adjudicator_list = label_list
+
+        label_text = ["contradict" if debates_majority_vote == 0 else "support"]
+        return label_text[0]