synthetic cases

app.py

@@ -1,15 +1,7 @@
 import gradio as gr
-import pandas as pd
-import numpy as np
-from rouge_score import rouge_scorer
-from joblib import Parallel, delayed
-from selfrank.algos.greedy import SelfRankGreedy
-from selfrank.algos.iterative import SelfRank
-from selfrank.algos.baseline import MCARank
-from selfrank.algos.triplet import equality, rouge
-import matplotlib.pyplot as plt
-from itertools import zip_longest
+from executors import benchmark_executor, synth_executor
 
+from gradio_rangeslider import RangeSlider
 class UI:
 
     def __init__(self):
@@ -32,36 +24,37 @@ class UI:
         gr.Markdown(
             """Using inference data gathered from [HELM](https://crfm.stanford.edu/helm/classic/latest/) we first show how our estimated rankings compare to rankings derived from using ground-truth or reference data."""
         )
+        
         with gr.Column(variant="compact"):
             self.data = gr.Dropdown(
-                choices=["CNN/DM", "XSUM", "MMLU"],
-                multiselect=False,
-                value="CNN/DM",
-                label="Choose a dataset.",
-                info="The dataset describes a specific task, either summarization (CNN/DM, XSUM) or multiple choice (MMLU).",
-                interactive=True,
-            )
+                    choices=["CNN/DM", "XSUM", "MMLU"],
+                    multiselect=False,
+                    value="CNN/DM",
+                    label="Choose a dataset.",
+                    info="The dataset describes a specific task, either summarization (CNN/DM, XSUM) or multiple choice (MMLU).",
+                    interactive=True,
+                )
             self.mmlu = gr.Dropdown(visible=False)
             self.evaluation = gr.Dropdown(
-                choices=["Rouge", "Equality"],
-                multiselect=False,
-                value="Rouge",
-                interactive=True,
-                label="Evaluation function",
-                info="How should the Judge model decide the winner? Demo limited to use 'Rouge' for generative tasks like summarization, and 'equality' for multiple choice or classification tasks. In practice you can use any function that compares judge responses to the contestant models.",
-            )
+                    choices=["Rouge", "Equality"],
+                    multiselect=False,
+                    value="Rouge",
+                    interactive=True,
+                    label="Evaluation function",
+                    info="How should the Judge model decide the winner? Demo limited to use 'Rouge' for generative tasks like summarization, and 'equality' for multiple choice or classification tasks. In practice you can use any function that compares judge responses to the contestant models.",
+                )
 
             def update_mmlu(v):
                 if v == "MMLU":
                     return gr.Dropdown(
-                        choices=list(['abstract_algebra', 'college_chemistry', 'computer_security', 'econometrics', 'us_foreign_policy']),
-                        value='us_foreign_policy',
-                        multiselect=False,
-                        label="Choose MMLU subject.",
-                        info="MMLU subject area.",
-                        interactive=True,
-                        visible=True,
-                    ), gr.Dropdown(choices=['Equality'], value='Equality')
+                            choices=list(['abstract_algebra', 'college_chemistry', 'computer_security', 'econometrics', 'us_foreign_policy']),
+                            value='us_foreign_policy',
+                            multiselect=False,
+                            label="Choose MMLU subject.",
+                            info="MMLU subject area.",
+                            interactive=True,
+                            visible=True,
+                        ), gr.Dropdown(choices=['Equality'], value='Equality')
                 else:
                     return gr.Dropdown(visible=False), gr.Dropdown(choices=['Rouge'], value='Rouge')
 
@@ -69,30 +62,31 @@ class UI:
 
 
             self.nmodels = gr.Dropdown(
-                choices=["All", 10, 20, 30],
-                label="Number of models",
-                info="Sample a subset of LLMs to rank.",
-                value=10,
-                interactive=True,
-            )
+                    choices=["All", 10, 20, 30],
+                    label="Number of models",
+                    info="Sample a subset of LLMs to rank.",
+                    value=10,
+                    interactive=True,
+                )
             self.nrows = gr.Dropdown(
-                choices=["All", 10, 20, 30],
-                label="Number of instances",
-                info="Sample a subset of instances to evaluate (smaller is faster).",
-                value=10,
-                interactive=True,
-            )
+                    choices=["All", 10, 20, 30],
+                    label="Number of instances",
+                    info="Sample a subset of instances to evaluate (smaller is faster).",
+                    value=10,
+                    interactive=True,
+                )
             self.method = gr.Dropdown(
-                choices=["Greedy", "Full"],
-                label="Algorithm variant to use",
-                info="Choose from one of two variants. 'Full' (FTR in the paper) runs all triplet combinations, recommended when evaluations are cheap or for smaller datasets, or 'greedy' (GTR) a faster variant suggested for more complex evaluations.",
-                value="Full",
-                interactive=True,
-            )
+                    choices=["Greedy", "Full"],
+                    label="Algorithm variant to use",
+                    info="Choose from one of two variants. 'Full' (FTR in the paper) runs all triplet combinations, recommended when evaluations are cheap or for smaller datasets, or 'greedy' (GTR) a faster variant suggested for more complex evaluations.",
+                    value="Full",
+                    interactive=True,
+                )
             self.btn_execute = gr.Button("Run")
 
     def output_panel(self):
         """Plots/leaderboard/bump charts"""
+    
         with gr.Column(variant="default"):
             gr.Markdown("""<h2 style='color: purple;'> Estimated ranking </h2> """)
             self.leaderboard = gr.DataFrame(headers=["rank", "model"],
@@ -110,8 +104,47 @@ class UI:
         """Synthetic data experiments"""
         gr.Markdown("<br>")
         gr.Markdown("---")
-        gr.Markdown("""<h1 style='color: purple;'>Synthetic multiple choice </h1> """)
-        gr.Markdown("Coming soon.")
+        with open("assets/synth.md", "r") as f:
+            content = f.read()
+
+        gr.Markdown(content)
+        
+        with gr.Row():
+            with gr.Column(scale=1):
+                with gr.Column(variant='compact'):
+                    self.synth_range = RangeSlider(10, 100, value=(50, 90), step=1, label="Model Accuracy Range (%)", interactive=True)
+                    self.synth_nmodels = gr.Slider(3, 50, value=10, step=1, label="Number of models to synthesise.", info="Equally spaced in the accuracy range.", interactive=True)
+                    self.synth_nanswers = gr.Slider(2, 50, value=10, step=1, label="Number of possible (discrete) answers per prompt.", interactive=True)
+                    self.synth_nquestions = gr.Slider(10, 100, step=10, label="Number of prompts to simulate.", interactive=True)
+                    self.synth_noise = gr.Slider(0, 1, value=0, label='Noise in evaluation (p)', info="Evaluation function decisions flipped with probability p. p=0 implies no noise.", interactive=True)
+                    self.synth_method = gr.Dropdown(
+                choices=["Greedy", "Full"],
+                label="Algorithm variant to use",
+                info="Choose from one of two variants. 'Full' (FTR in the paper) runs all triplet combinations, recommended when evaluations are cheap or for smaller datasets, or 'greedy' (GTR) a faster variant suggested for more complex evaluations.",
+                value="Full",
+                interactive=True,
+            )
+
+                    examples = gr.Examples([[(10, 30), 10, 10, 10, 0, "Full"], 
+                                            [(10, 30), 10, 10, 10, 0.5, "Full"], 
+                                            [[10, 30], 10, 2, 10, 0, "Full"]], 
+                                           [self.synth_range, self.synth_nmodels, self.synth_nanswers, self.synth_nquestions, self.synth_noise, self.synth_method],
+                                             label='Some interesting cases (click and run)', example_labels=["Rankings recovered for low accuracy models", 
+                                                                                    "Robust recovery when evaluations have noise",
+                                                                                    "Binary outcomes are challenging"
+                                                                                    ] )
+
+                    self.synth_execute = gr.Button("Run")
+            with gr.Column(scale=1):
+                with gr.Column(variant="default"):
+                    gr.Markdown(
+                        """<h2 style='color: purple;'> Estimated vs. true ranking </h2> """
+                    )
+                    
+                    self.synth_bumpchart = gr.Image()
+            with gr.Column(scale=1):
+                self.synth_eval_metrics = gr.Markdown()
+
 
     def byod_panel(self):
         """Instructions panel"""
@@ -141,12 +174,12 @@ class UI:
                 # Output panel/leaderboard
                 self.output_panel()
 
-            # TODO: self.synth_panel()
+            self.synth_panel()
             self.byod_panel()
 
             # Register event listeners
             self.btn_execute.click(
-                fn=self.benchmark_executor,
+                fn=benchmark_executor,
                 inputs=[
                     self.data,
                     self.mmlu,
@@ -157,145 +190,21 @@ class UI:
                 ],
                 outputs=[self.leaderboard, self.bumpchart, self.eval_metrics],
             )
-
-        return demo
-
-    def benchmark_executor(
-        self, data, mmlu_subject, evaluation, nmodels, nrows, method
-    ) -> tuple[pd.DataFrame, plt.figure]:
-        """Main execution flow for benchmarks"""
-
-        # gr.Info(f"Loaded run config: {data}, {evaluation}, {nmodels}.")
-        seed = 40
-        np.random.seed(seed)
-
-        match data:
-            case "MMLU":
-                adf = pd.read_pickle(f"data/mmlu_subject_{mmlu_subject}.pkl")
-
-            case "CNN/DM":
-                adf = pd.read_pickle(f"data/cnndm.pkl")
-
-            case "XSUM":
-                adf = pd.read_pickle(f"data/xsum.pkl")
-
-            case _:
-                raise ValueError(f"'{data}' not understood.")
-
-        MODELS = adf.model.unique()
-
-        # Sample fewer models if so needed
-        if nmodels != "All":
-            if nmodels < len(MODELS):
-
-                MODELS = np.random.choice(MODELS, nmodels, replace=False).tolist()
-                adf = adf[adf.model.isin(MODELS)]
-
-        match data:
-            case "MMLU":
-                keys = [
-                    "id",
-                    "trial_id",
-                    "perturbation",
-                ]  # MMLU has this extra parameter
-            case "CNN/DM" | "XSUM":
-                keys = ["id", "trial_id"]
-            case _:
-                pass
-
-        df = adf.pivot_table(
-            columns="model",
-            index=keys,
-            values="output",
-            aggfunc="first",
-        )
-
-        # Filter by number of rows
-        df.dropna(inplace=True)
-        if nrows != "All":
-            if nrows < df.shape[0]:
-                df = df.sample(nrows, random_state=seed)
-
-        # Compute true ranking
-        adf = adf.set_index(keys).loc[df.index].reset_index()
-
-        if evaluation == "Rouge":
-
-            def __true_rouge(x, scorer):
-                return scorer.score(x["reference"], x["output"])["rouge2"].fmeasure
-
-            scorer = rouge_scorer.RougeScorer(["rouge2"], use_stemmer=True)
-            adf["rouge"] = Parallel(n_jobs=-1, batch_size=128)(
-                delayed(__true_rouge)(i, scorer) for _, i in adf.iterrows()
-            )
-
-            # Method 2 - look at "win rates" - for each question, see which model
-            # wins (i.e. has the best ROUGE score)
-            idx = adf.groupby(["id", "trial_id"])["rouge"].idxmax()
-            win_rates = adf.loc[idx].model.value_counts()
-            win_rate_rank = win_rates.index.tolist()
-
-            # include models with nowins at the bottom
-            no_wins = list(set(MODELS) - set(win_rate_rank))
-            true_ranking = win_rate_rank + no_wins
-            evaluator = rouge
-
-        elif evaluation == "Equality":
-
-            # Compute the true ranking (multiple choice - so use equality between
-            # LLM response and reference-value)
-            adf["C"] = (adf.output == adf.reference).astype(int)
-            true_ranking = (
-                adf.groupby("model")["C"]
-                .apply(lambda x: sum(x) / len(x))
-                .sort_values(ascending=False)
-                .index.tolist()
+            self.synth_execute.click(
+                fn=synth_executor,
+                inputs=[
+                    self.synth_range,
+                    self.synth_nmodels,
+                    self.synth_nanswers,
+                    self.synth_nquestions,
+                    self.synth_noise,
+                    self.synth_method,
+                ],
+                outputs=[self.synth_bumpchart, self.synth_eval_metrics],
             )
-            evaluator = equality
-
-        else:
-            raise ValueError(f"'{evaluation}' not understood.")
-
-        match method:
-            case "Full":
-                ranker = SelfRank(MODELS, evaluator, true_ranking)
-
-            case "Greedy":
-                ranker = SelfRankGreedy(MODELS, evaluator, true_ranking)
-
-            case "MCA":
-                raise NotImplementedError
-            case _:
-                raise ValueError(f"'{method}' not understood.")
-
-        # generate outputs
-        ranker.fit(df)
-        ranks = ranker.ranking
-        
-        ranks = [
-            j + i for i, j in zip_longest(ranks, ["🥇 ", "🥈 ", "🥉 "], fillvalue="")
-        ]
-        out_df = pd.DataFrame({"rank": range(1, len(true_ranking) + 1), "model": ranks})
-
-        out_metrics = {
-            "rbo": ranker.measure(metric="rbo"),
-            "map-1": ranker.measure(metric="mapk", k=1),
-            "map-3": ranker.measure(metric="mapk", k=3),
-            "map-5": ranker.measure(metric="mapk", k=5),
-            "map-10": ranker.measure(metric="mapk", k=10),
-            "evaluations": evaluator.calls,
-        }
-        eval_metrics = (
-            f"<h2> Evaluation measures </h2>"
-            f"Rank-Biased Overlap: {out_metrics['rbo']:0.3f}<br>"
-            f"MAP-3              : {out_metrics['map-3']:0.3f}<br>"
-            f"MAP-5              : {out_metrics['map-5']:0.3f}<br>"
-            f"MAP-10             : {out_metrics['map-10']: 0.3f}."
-        )
-
-        out_plot = ranker.plot()
+        return demo
 
-        return out_df, "output.png", eval_metrics
+   
 
     def run(self):
         self.ui = self.layout()

assets/synth.md

@@ -0,0 +1,3 @@
+<h1 style='color: purple;'>Synthetic multiple choice </h1>
+
+To analyse our methods, we synthesise data from models with known accuracy in a multiple choice setting, i.e. discrete set of possible responses. Several parameters (number of models, model accuracy, number of prompts, and number of possible answers, noisy comparisons) can have an impact on quality of results. Rankings can be recovered for a range of challenging cases, for instance when the accuracy of underlying models is low or when the evaluation function is noisy and imperfect. When the number of possible answers are low, for example in binary choice settings, recovering rankings becomes challenging. In general low variance in wrong answers cause triplet evaluations to treat wrong answers as the right one. 

executors.py

@@ -0,0 +1,272 @@
+
+import pandas as pd
+import numpy as np
+from rouge_score import rouge_scorer
+from joblib import Parallel, delayed
+from selfrank.algos.greedy import SelfRankGreedy
+from selfrank.algos.iterative import SelfRank
+from selfrank.algos.baseline import MCARank
+from selfrank.algos.triplet import equality, rouge, noisy_equality
+import matplotlib.pyplot as plt
+from itertools import zip_longest
+from uuid import uuid4
+import csv, os
+from functools import partial
+
+
+def generate_data(max_acc, min_acc, nmodels, nanswers, nquestions) -> tuple[pd.DataFrame, list]:
+
+    np.random.seed(42)
+    # Spread model accuracies between min and max
+    model_acc = np.linspace(max_acc, min_acc, nmodels)
+
+    gt_and_model_ans = np.zeros(
+        (nquestions, nmodels + 1), dtype=int
+    )  # array to store ground truth and model ans
+
+    # Create ground truth answers i.e. first column
+    for i in range(nquestions):
+        gt_and_model_ans[i][0] = np.random.randint(nanswers)
+
+    for i in range(0, nmodels):
+        no_of_entries_frm_gt = np.ceil(model_acc[i] / 100 * (nquestions)).astype(int)
+        # print(no_of_entries_frm_gt)
+        offsets_to_match = np.random.permutation(nquestions)[0:no_of_entries_frm_gt]
+        # print(offsets_to_match)
+        for j in range(nquestions):
+            if j in offsets_to_match:
+                gt_and_model_ans[j][i + 1] = gt_and_model_ans[j][0]
+            else:
+                lst_wo_gt = list(range(nanswers))
+                lst_wo_gt.remove(gt_and_model_ans[j][0])
+                gt_and_model_ans[j][i + 1] = lst_wo_gt[np.random.randint(nanswers - 1)]
+
+    # print(gt_and_model_ans)
+    filename = str(uuid4())
+
+    fields = ["GT"]
+    for i in range(nmodels):
+        fields.append("M" + str(i + 1))
+
+    # writing to csv file
+    with open(filename, "w") as csvfile:
+        # creating a csv writer object
+        csvwriter = csv.writer(csvfile)
+
+        # writing the fields
+        csvwriter.writerow(fields)
+
+        # writing the data rows
+        csvwriter.writerows(gt_and_model_ans)
+
+    df = pd.read_csv(filename)
+    os.remove(filename)
+
+    true_ranking = [f"M{i}" for i in range(1, nmodels + 1)]
+
+    return df, true_ranking
+
+def synth_executor(acc_range: tuple[float, float], nmodels, nanswers, nquestions, noise, method) -> tuple[str, dict]:
+
+    min_acc, max_acc = acc_range
+    df, true_ranking = generate_data(max_acc, min_acc, nmodels, nanswers, nquestions)
+
+    if noise == 0.:
+        comp = equality
+    else:
+        comp = partial(noisy_equality, p=noise)
+    
+    df = df.drop(columns=["GT"])
+    MODELS = df.columns.tolist()
+
+    if method == "Full":
+        ranker = SelfRank(MODELS, comp, true_ranking)
+        ranker.fit(df)
+
+        # outputs of interest
+        out = {
+            "true_ranking": true_ranking,
+            "estimated_ranking": ranker.ranking,
+            "rbo": ranker.measure(metric="rbo"),
+            "map-1": ranker.measure(metric='mapk', k=1),
+            "map-3": ranker.measure(metric='mapk', k=3),
+            "map-5": ranker.measure(metric='mapk', k=5),
+            "map-10": ranker.measure(metric='mapk', k=10)
+        }
+
+    elif method == "Greedy":
+        ranker = SelfRankGreedy(MODELS, comp, true_ranking)
+        ranker.fit(df)
+        out = {
+            "true_ranking": true_ranking,
+            "estimated_ranking": ranker.ranking,
+            "rbo": ranker.measure(metric="rbo"),
+            "map-1": ranker.measure(metric='mapk', k=1),
+            "map-3": ranker.measure(metric='mapk', k=3),
+            "map-5": ranker.measure(metric='mapk', k=5),
+            "map-10": ranker.measure(metric='mapk', k=10)
+        }
+    elif method == 'MCA':
+        ranker = MCARank(MODELS, comp, true_ranking)
+        ranker.fit(df, measure='noisy_equality', p=noise)
+        out = {
+            "true_ranking": true_ranking,
+            "estimated_ranking": ranker.ranking,
+            "rbo": ranker.measure(metric="rbo"),
+            "map-1": ranker.measure(metric='mapk', k=1),
+            "map-3": ranker.measure(metric='mapk', k=3),
+            "map-5": ranker.measure(metric='mapk', k=5),
+            "map-10": ranker.measure(metric='mapk', k=10)
+        }
+    else:
+        raise ValueError(f"{method} not understood.")
+
+    eval_metrics = (
+            f"<h2 style='color: purple;'> Evaluation measures </h2>"
+            f"Rank-Biased Overlap: {out['rbo']:0.3f}<br>"
+            f"MAP-3              : {out['map-3']:0.3f}<br>"
+            f"MAP-5              : {out['map-5']:0.3f}<br>"
+            f"MAP-10             : {out['map-10']: 0.3f}."
+        )
+
+    out_plot = ranker.plot("synth")
+
+    return "synth.png", eval_metrics
+
+
+
+def benchmark_executor(data, mmlu_subject, evaluation, nmodels, nrows, method
+    ) -> tuple[pd.DataFrame, plt.figure]:
+        """Main execution flow for benchmarks"""
+
+        # gr.Info(f"Loaded run config: {data}, {evaluation}, {nmodels}.")
+        seed = 40
+        np.random.seed(seed)
+
+        match data:
+            case "MMLU":
+                adf = pd.read_pickle(f"data/mmlu_subject_{mmlu_subject}.pkl")
+
+            case "CNN/DM":
+                adf = pd.read_pickle(f"data/cnndm.pkl")
+
+            case "XSUM":
+                adf = pd.read_pickle(f"data/xsum.pkl")
+
+            case _:
+                raise ValueError(f"'{data}' not understood.")
+
+        MODELS = adf.model.unique()
+
+        # Sample fewer models if so needed
+        if nmodels != "All":
+            if nmodels < len(MODELS):
+
+                MODELS = np.random.choice(MODELS, nmodels, replace=False).tolist()
+                adf = adf[adf.model.isin(MODELS)]
+
+        match data:
+            case "MMLU":
+                keys = [
+                    "id",
+                    "trial_id",
+                    "perturbation",
+                ]  # MMLU has this extra parameter
+            case "CNN/DM" | "XSUM":
+                keys = ["id", "trial_id"]
+            case _:
+                pass
+
+        df = adf.pivot_table(
+            columns="model",
+            index=keys,
+            values="output",
+            aggfunc="first",
+        )
+
+        # Filter by number of rows
+        df.dropna(inplace=True)
+        if nrows != "All":
+            if nrows < df.shape[0]:
+                df = df.sample(nrows, random_state=seed)
+
+        # Compute true ranking
+        adf = adf.set_index(keys).loc[df.index].reset_index()
+
+        if evaluation == "Rouge":
+
+            def __true_rouge(x, scorer):
+                return scorer.score(x["reference"], x["output"])["rouge2"].fmeasure
+
+            scorer = rouge_scorer.RougeScorer(["rouge2"], use_stemmer=True)
+            adf["rouge"] = Parallel(n_jobs=-1, batch_size=128)(
+                delayed(__true_rouge)(i, scorer) for _, i in adf.iterrows()
+            )
+
+            # Method 2 - look at "win rates" - for each question, see which model
+            # wins (i.e. has the best ROUGE score)
+            idx = adf.groupby(["id", "trial_id"])["rouge"].idxmax()
+            win_rates = adf.loc[idx].model.value_counts()
+            win_rate_rank = win_rates.index.tolist()
+
+            # include models with nowins at the bottom
+            no_wins = list(set(MODELS) - set(win_rate_rank))
+            true_ranking = win_rate_rank + no_wins
+            evaluator = rouge
+
+        elif evaluation == "Equality":
+
+            # Compute the true ranking (multiple choice - so use equality between
+            # LLM response and reference-value)
+            adf["C"] = (adf.output == adf.reference).astype(int)
+            true_ranking = (
+                adf.groupby("model")["C"]
+                .apply(lambda x: sum(x) / len(x))
+                .sort_values(ascending=False)
+                .index.tolist()
+            )
+            evaluator = equality
+
+        else:
+            raise ValueError(f"'{evaluation}' not understood.")
+
+        match method:
+            case "Full":
+                ranker = SelfRank(MODELS, evaluator, true_ranking)
+
+            case "Greedy":
+                ranker = SelfRankGreedy(MODELS, evaluator, true_ranking)
+
+            case "MCA":
+                raise NotImplementedError
+            case _:
+                raise ValueError(f"'{method}' not understood.")
+
+        # generate outputs
+        ranker.fit(df)
+        ranks = ranker.ranking
+        
+        ranks = [
+            j + i for i, j in zip_longest(ranks, ["🥇 ", "🥈 ", "🥉 "], fillvalue="")
+        ]
+        out_df = pd.DataFrame({"rank": range(1, len(true_ranking) + 1), "model": ranks})
+
+        out_metrics = {
+            "rbo": ranker.measure(metric="rbo"),
+            "map-1": ranker.measure(metric="mapk", k=1),
+            "map-3": ranker.measure(metric="mapk", k=3),
+            "map-5": ranker.measure(metric="mapk", k=5),
+            "map-10": ranker.measure(metric="mapk", k=10),
+            "evaluations": evaluator.calls,
+        }
+        eval_metrics = (
+            f"<h2 style='color: purple;'> Evaluation measures </h2>"
+            f"Rank-Biased Overlap: {out_metrics['rbo']:0.3f}<br>"
+            f"MAP-3              : {out_metrics['map-3']:0.3f}<br>"
+            f"MAP-5              : {out_metrics['map-5']:0.3f}<br>"
+            f"MAP-10             : {out_metrics['map-10']: 0.3f}."
+        )
+
+        out_plot = ranker.plot()
+
+        return out_df, "output.png", eval_metrics

selfrank/algos/triplet.py

@@ -9,6 +9,7 @@ import logging
 from .plots import bcolors
 import random
 
+
 logger = logging.getLogger(__name__)
 
 # Local only for now
@@ -114,7 +115,7 @@ def equality(a: str, b:str, c:str, df:pd.DataFrame) -> int:
 
     simple heuristic as the answers are multiple choice, so use equality.
     """
-
+    
     ties = df[a] == df[b]
     a_wins = sum((df[a] == df[c]) & ~(ties))
     b_wins = sum((df[b] == df[c]) & ~(ties))
@@ -132,7 +133,7 @@ def noisy_equality(a: str, b:str, c:str, df:pd.DataFrame, p: float) -> int:
     noisy version of equality - where evaluations are flipped independently with
     probability p (p=1 will always flip, p=0, will never)
     """
-
+    random.seed(42)
     perturb = lambda x: not x if (random.random() <= p) else x
 
     ties = (df[a] == df[b])