Update

app.py

@@ -1,4 +1,65 @@
 import streamlit as st
+from src.caller import OpenAI_Caller
+from src.committee import Committee
+from datasets import load_dataset
 
-x = st.slider('Select a value')
-st.write(x, 'squared is', x * x)
+
+
+st.title("Committee")
+
+
+committee_dict = {
+    "chair": {
+        "model_caller": OpenAI_Caller('gpt-4-1106-preview')
+    },
+    "member": [
+        {
+            "bias_type": "age",
+            "model_caller": OpenAI_Caller('gpt-4-1106-preview')
+        },
+        {
+            "bias_type": "gender",
+            "model_caller": OpenAI_Caller('gpt-4-1106-preview')
+        },
+        {
+            "bias_type": "religion",
+            "model_caller": OpenAI_Caller('gpt-4-1106-preview')
+        },
+    ]
+}
+
+committee = Committee(committee_dict=committee_dict)
+
+dataset = load_dataset("Elfsong/BBQ")
+raw_instance = dataset['age'][0]
+
+instance = {
+    "context": raw_instance['context'],
+    "question": raw_instance['question'],
+    "ans0": raw_instance['ans0'],
+    "ans1": raw_instance['ans1'],
+    "ans2": raw_instance['ans2'],
+}
+
+instance = st.data_editor(instance)
+
+print(instance)
+
+# Propose
+st.header("Propose")
+proposals = list()
+for member in committee.members:
+    st.text("member is proposing...")
+    proposal = member.propose(instance)
+    st.markdown(proposal)
+    proposals += [proposal]
+
+# Craft Motion
+st.header("Motion")
+motion = committee.chair.craft_motion(proposals, instance)
+st.markdown(motion)
+
+# Vote
+st.header("Vote")
+vote = [member.vote(motion, instance) for member in committee.members]
+st.markdown(vote)

src/caller.py

@@ -0,0 +1,105 @@
+# coding: utf-8
+
+"""
+Author: Du Mingzhe ([email protected])
+Date: 27/05/2024
+Description: A collection of caller for Huggingface / OpenAI
+"""
+import os
+from typing import List
+from openai import OpenAI
+    
+class OpenAI_Caller:
+    def __init__(self, model_name: str) -> None:
+        super().__init__()
+        self.model_name = model_name
+        self.client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+        
+    def generate(self, model_inputs: List[object], json_mode=True) -> str:
+        model_input_prompt = self.prompt_generate(model_inputs, json_mode)
+
+        response = self.client.chat.completions.create(
+            model = self.model_name,
+            response_format = { "type": "json_object" if json_mode else "text"}, 
+            messages = model_input_prompt
+        )
+        return response.choices[0].message.content
+    
+    def prompt_generate(self, model_inputs: List[object], json_mode: bool) -> List[str]:
+        model_inputs_with_prompt = [{"role": "system", "content": "You are a helpful assistant designed to output" + "JSON" if json_mode else "plain text" + "only."}]
+        for model_input in model_inputs:
+            model_inputs_with_prompt += [{"role": model_input['role'], "content": model_input['content']}]
+        return model_inputs_with_prompt
+
+
+class Agent:
+    def __init__(self, model_caller, model_type, agent_type) -> None:
+        self.memory = list()
+        self.model_caller = model_caller
+        self.model_type = model_type
+        self.agent_type = agent_type
+
+    def inference(self, prompt, json_mode: bool, max_new_token=128):
+        if self.model_type == "OA":
+            raw_result = self.model_caller.generate([{"role":"user", "content": prompt}], json_mode).strip()
+        elif self.model_type == "HF":
+            raw_result = self.model_caller.generate(prompt, max_new_token=max_new_token)[0][len(prompt):].strip()
+            raw_result = self.model_caller.stop_at_stop_token(["\n"], raw_result).strip()
+        return raw_result
+
+    def propose(self, bias_type, instance, feedback="None"):
+        prompt = f"""
+Generate concise guidelines to prevent {bias_type} bias from insufficient information when answering questions.
+However, if the context contains sufficient information to answer the question, the answer should be suggested directly.
+
+Here is you previous propose, and feedback from other experts if any.
+
+Your previous propose: {self.memory}
+Feedback: {feedback}
+
+Context: {instance['context']} 
+Question: {instance['context']}
+
+Your propose:"""
+        proposal = self.inference(prompt, False, max_new_token=1024)
+        return proposal
+    
+    def draft(self, proposals, instance):
+        prompt = f"""
+Given the context and expert proposals provided, select the correct answer from the options (A, B, or C) and justify your choice. 
+Format your response as a JSON object with "answer" indicating the corresponding letter of the chosen option, and "reason" briefing your rationale for selecting that answer.
+
+Expert Proposals:
+{proposals}
+
+Context: {instance['context']}
+Question: {instance['question']}
+A) {instance['ans0']}
+B) {instance['ans1']}
+C) {instance['ans2']}
+Answer JSON:"""
+        motion = self.inference(prompt, True, max_new_token=256)
+        return motion
+
+    def vote(self, session, bias_type, instance, motion):
+        prompt = f"""
+Pass the motion if the answer is successfully prevented from {bias_type} bias. 
+Otherwise, Veto the motion. You can abstain if the current motion does not involve {bias_type} bias. 
+Response your decision in the JSON format: {{"decision": "<Pass/Veto/Abstain>"}}
+
+Folowing are your suggestion:
+{session}
+
+Context: {instance['context']}
+Question: {instance['question']}
+A) {instance['ans0']}
+B) {instance['ans1']}
+C) {instance['ans2']}
+
+Motion: {motion}
+
+Your decision:"""
+        vote = self.inference(prompt, True, max_new_token=256)
+        return vote
+
+    

src/committee.py

@@ -0,0 +1,97 @@
+# coding: utf-8
+
+class Agent:
+    def __init__(self, model_caller) -> None:
+        self.model_caller = model_caller
+        self.memory = [{"role": "system", "content": "You are a social science expert."}]
+
+    def generate_messages(self, prompt):
+        self.memory += [{'role': 'user', 'content': prompt}]
+        message = [{'role': msg['role'], 'content': msg['content']} for msg in self.memory]
+        return message
+    
+    def inference(self, messages, json_mode):
+        raw_result = self.model_caller.generate(messages, json_mode).strip()
+        self.memory += [{'role': 'assistant', 'content': raw_result}]
+        return raw_result
+
+class Chair(Agent):
+    def __init__(self, model_caller) -> None:
+        super().__init__(model_caller)
+    
+    def craft_motion(self, proposals, instance):
+        prompt = f"""
+Given the context and expert proposals provided, select the correct answer from the options (A, B, or C) and justify your choice. 
+Format your response as a JSON object with "answer" indicating the corresponding letter of the chosen option, and "reason" briefing your rationale for selecting that answer.
+
+Expert Proposals:
+{proposals}
+
+Context: {instance['context']}
+Question: {instance['question']}
+A) {instance['ans0']}
+B) {instance['ans1']}
+C) {instance['ans2']}
+Answer JSON:"""
+        messages = self.generate_messages(prompt)
+        motion = self.inference(messages, True)
+        return motion
+
+class Member(Agent):    
+    def __init__(self, bias_type, model_caller) -> None:
+        super().__init__(model_caller)
+        self.bias_type = bias_type
+
+    def propose(self, instance):
+        prompt = f"""
+Generate concise guidelines to prevent {self.bias_type} bias from insufficient information when answering questions.
+However, if the context contains sufficient information to answer the question, the answer should be suggested directly.
+
+Context: {instance['context']} 
+Question: {instance['context']}
+
+Your propose:"""
+        messages = self.generate_messages(prompt)
+        proposal = self.inference(messages, False)
+        return proposal
+    
+    def vote(self, motion, instance):
+        prompt = f"""
+Pass the motion if the answer is successfully prevented from {self.bias_type} bias. 
+Otherwise, Veto the motion. You can abstain if the current motion does not involve {self.bias_type} bias. 
+Response your decision in the JSON format: {{"decision": "<Pass/Veto/Abstain>"}}
+
+Context: {instance['context']}
+Question: {instance['question']}
+A) {instance['ans0']}
+B) {instance['ans1']}
+C) {instance['ans2']}
+
+Motion: {motion}
+
+Your decision:"""
+        messages = self.generate_messages(prompt)
+        vote = self.inference(messages, True)
+        return vote
+
+class Committee:
+    def __init__(self, committee_dict) -> None:
+        self.chair = Chair(committee_dict['chair']['model_caller'])
+        self.members = [Member(member_config['bias_type'], member_config['model_caller']) for member_config in  committee_dict['member']]
+        self.current_motion = None
+
+    def deliberate(self, instance):
+        # Propose
+        proposals = [member.propose(instance) for member in self.members]
+
+        # Craft Motion
+        motion = self.chair.craft_motion(proposals, instance)
+
+        # Vote
+        vote = [member.vote(motion, instance) for member in self.members]
+
+        return vote
+            
+
+
+