from collections.abc import Sequence import random import gradio as gr # If the watewrmark is not detected, consider the use case. Could be because of # the nature of the task (e.g., fatcual responses are lower entropy) or it could # be another _GEMMA_2B = 'google/gemma-2b' _PROMPTS: tuple[str] = ( 'prompt 1', 'prompt 2', 'prompt 3', 'prompt 4', ) _CORRECT_ANSWERS: dict[str, bool] = {} with gr.Blocks() as demo: prompt_inputs = [ gr.Textbox(value=prompt, lines=4, label='Prompt') for prompt in _PROMPTS ] generate_btn = gr.Button('Generate') with gr.Column(visible=False) as generations_col: generations_grp = gr.CheckboxGroup( label='All generations, in random order', info='Select the generations you think are watermarked!', ) reveal_btn = gr.Button('Reveal', visible=False) with gr.Column(visible=False) as detections_col: revealed_grp = gr.CheckboxGroup( label='Ground truth for all generations', info=( 'Watermarked generations are checked, and your selection are ' 'marked as correct or incorrect in the text.' ), ) detect_btn = gr.Button('Detect', visible=False) def generate(*prompts): standard = [f'{prompt} response' for prompt in prompts] watermarked = [f'{prompt} watermarked response' for prompt in prompts] responses = standard + watermarked random.shuffle(responses) _CORRECT_ANSWERS.update({ response: response in watermarked for response in responses }) # Load model return { generate_btn: gr.Button(visible=False), generations_col: gr.Column(visible=True), generations_grp: gr.CheckboxGroup( responses, ), reveal_btn: gr.Button(visible=True), } generate_btn.click( generate, inputs=prompt_inputs, outputs=[generate_btn, generations_col, generations_grp, reveal_btn] ) def reveal(user_selections: list[str]): choices: list[str] = [] value: list[str] = [] for response, is_watermarked in _CORRECT_ANSWERS.items(): if is_watermarked and response in user_selections: choice = f'Correct! {response}' elif not is_watermarked and response not in user_selections: choice = f'Correct! {response}' else: choice = f'Incorrect. {response}' choices.append(choice) if is_watermarked: value.append(choice) return { reveal_btn: gr.Button(visible=False), detections_col: gr.Column(visible=True), revealed_grp: gr.CheckboxGroup(choices=choices, value=value), detect_btn: gr.Button(visible=True), } reveal_btn.click( reveal, inputs=generations_grp, outputs=[ reveal_btn, detections_col, revealed_grp, detect_btn ], ) if __name__ == '__main__': demo.launch()