add files

averitec/data/all_samples.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef79bab962c2b17d56eb2582b9919bfe8023858fa13ba20c591900857b561854
+size 11444395

averitec/data/sample_claims.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Created by zd302 at 09/05/2024
+
+
+CLAIMS_Type = {
+    "Claim": [
+        "England won the Euro 2024.",
+        "Albert Einstein works in the field of computer science.",
+    ],
+    "Event/Property Claim": [
+        'Hunter Biden had no experience in Ukraine or in the energy sector when he joined the board of Burisma.',
+        "After the police shooting of Jacob Blake, Gov. Tony Evers & Lt. Gov. Mandela Barnes did not call for peace or encourage calm.",
+        "President Trump fully co-operated with the investigation into Russian interference in the 2016 U.S presidential campaign.",
+    ],
+    "Causal Claim":[
+        "Anxiety levels among young teenagers dropped during the coronavirus pandemic, a study has suggested",
+        "Auto workers across Michigan could have lost their jobs if not for Barack Obama and Joe Biden",
+    ],
+    "Numerical Claim":[
+        "Sweden, despite never having had lockdown, has a lower COVID-19 death rate than Spain, Italy, and the United Kingdom.",
+        "According to Harry Roque, even if 10,000 people die, 10 million COVID-19 cases in the country will not be a loss.",
+    ]
+}
+
+CLAIMS_FACT_CHECKING_STRATEGY= {
+    "Written Evidence": [
+        "Pretty Little Thing's terms and conditions state that its products may contain chemicals that can cause cancer, birth defects or other reproductive harm.",
+        "Pretty Little Thing products may contain chemicals that can cause cancer, birth defects or other reproductive harm.",
+    ],
+    "Numerical Comparison":[
+        "Congress party claims regarding shortfall in Government earnings",
+        "On average, one person dies by suicide every 22 hours in West Virginia, United States.",
+    ],
+    "Consultation":[
+        "Your reaction to an optical illusion is an indication of your state of mind.",
+        "The last time people created a Hollywood blacklist, people ended up killing themselves. They were accused, and they lost their right to work.",
+    ]
+}

averitec/models/DualEncoderModule.py

@@ -0,0 +1,143 @@
+import pytorch_lightning as pl
+import torch
+from transformers.optimization import AdamW
+import torchmetrics
+
+
+class DualEncoderModule(pl.LightningModule):
+
+    def __init__(self, tokenizer, model, learning_rate=1e-3):
+        super().__init__()
+        self.tokenizer = tokenizer
+        self.model = model
+        self.learning_rate = learning_rate
+
+        self.train_acc = torchmetrics.Accuracy(
+            task="multiclass", num_classes=model.num_labels
+        )
+        self.val_acc = torchmetrics.Accuracy(
+            task="multiclass", num_classes=model.num_labels
+        )
+        self.test_acc = torchmetrics.Accuracy(
+            task="multiclass", num_classes=model.num_labels
+        )
+
+    def forward(self, input_ids, **kwargs):
+        return self.model(input_ids, **kwargs)
+
+    def configure_optimizers(self):
+        optimizer = AdamW(self.parameters(), lr=self.learning_rate)
+        return optimizer
+
+    def training_step(self, batch, batch_idx):
+        pos_ids, pos_mask, neg_ids, neg_mask = batch
+
+        neg_ids = neg_ids.view(-1, neg_ids.shape[-1])
+        neg_mask = neg_mask.view(-1, neg_mask.shape[-1])
+
+        pos_outputs = self(
+            pos_ids,
+            attention_mask=pos_mask,
+            labels=torch.ones(pos_ids.shape[0], dtype=torch.uint8).to(
+                pos_ids.get_device()
+            ),
+        )
+        neg_outputs = self(
+            neg_ids,
+            attention_mask=neg_mask,
+            labels=torch.zeros(neg_ids.shape[0], dtype=torch.uint8).to(
+                neg_ids.get_device()
+            ),
+        )
+
+        loss_scale = 1.0
+        loss = pos_outputs.loss + loss_scale * neg_outputs.loss
+
+        pos_logits = pos_outputs.logits
+        pos_preds = torch.argmax(pos_logits, axis=1)
+        self.train_acc(
+            pos_preds.cpu(), torch.ones(pos_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        neg_logits = neg_outputs.logits
+        neg_preds = torch.argmax(neg_logits, axis=1)
+        self.train_acc(
+            neg_preds.cpu(), torch.zeros(neg_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        return {"loss": loss}
+
+    def validation_step(self, batch, batch_idx):
+        pos_ids, pos_mask, neg_ids, neg_mask = batch
+
+        neg_ids = neg_ids.view(-1, neg_ids.shape[-1])
+        neg_mask = neg_mask.view(-1, neg_mask.shape[-1])
+
+        pos_outputs = self(
+            pos_ids,
+            attention_mask=pos_mask,
+            labels=torch.ones(pos_ids.shape[0], dtype=torch.uint8).to(
+                pos_ids.get_device()
+            ),
+        )
+        neg_outputs = self(
+            neg_ids,
+            attention_mask=neg_mask,
+            labels=torch.zeros(neg_ids.shape[0], dtype=torch.uint8).to(
+                neg_ids.get_device()
+            ),
+        )
+
+        loss_scale = 1.0
+        loss = pos_outputs.loss + loss_scale * neg_outputs.loss
+
+        pos_logits = pos_outputs.logits
+        pos_preds = torch.argmax(pos_logits, axis=1)
+        self.val_acc(
+            pos_preds.cpu(), torch.ones(pos_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        neg_logits = neg_outputs.logits
+        neg_preds = torch.argmax(neg_logits, axis=1)
+        self.val_acc(
+            neg_preds.cpu(), torch.zeros(neg_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        self.log("val_acc", self.val_acc)
+
+        return {"loss": loss}
+
+    def test_step(self, batch, batch_idx):
+        pos_ids, pos_mask, neg_ids, neg_mask = batch
+
+        neg_ids = neg_ids.view(-1, neg_ids.shape[-1])
+        neg_mask = neg_mask.view(-1, neg_mask.shape[-1])
+
+        pos_outputs = self(
+            pos_ids,
+            attention_mask=pos_mask,
+            labels=torch.ones(pos_ids.shape[0], dtype=torch.uint8).to(
+                pos_ids.get_device()
+            ),
+        )
+        neg_outputs = self(
+            neg_ids,
+            attention_mask=neg_mask,
+            labels=torch.zeros(neg_ids.shape[0], dtype=torch.uint8).to(
+                neg_ids.get_device()
+            ),
+        )
+
+        pos_logits = pos_outputs.logits
+        pos_preds = torch.argmax(pos_logits, axis=1)
+        self.test_acc(
+            pos_preds.cpu(), torch.ones(pos_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        neg_logits = neg_outputs.logits
+        neg_preds = torch.argmax(neg_logits, axis=1)
+        self.test_acc(
+            neg_preds.cpu(), torch.zeros(neg_ids.shape[0], dtype=torch.uint8).cpu()
+        )
+
+        self.log("test_acc", self.test_acc)

averitec/models/JustificationGenerationModule.py

@@ -0,0 +1,193 @@
+import pytorch_lightning as pl
+import torch
+import numpy as np
+import datasets
+from transformers import MaxLengthCriteria, StoppingCriteriaList
+from transformers.optimization import AdamW
+import itertools
+from averitec.models.utils import count_stats, f1_metric, pairwise_meteor
+from torchmetrics.text.rouge import ROUGEScore
+import torch.nn.functional as F
+import torchmetrics
+from torchmetrics.classification import F1Score
+
+def freeze_params(model):
+  for layer in model.parameters():
+    layer.requires_grade = False
+
+class JustificationGenerationModule(pl.LightningModule):
+    
+  def __init__(self, tokenizer, model, learning_rate=1e-3, gen_num_beams=2, gen_max_length=100, should_pad_gen=True):
+    super().__init__()
+    self.tokenizer = tokenizer
+    self.model = model
+    self.learning_rate = learning_rate
+
+    self.gen_num_beams = gen_num_beams
+    self.gen_max_length = gen_max_length
+    self.should_pad_gen = should_pad_gen
+
+    #self.metrics =  datasets.load_metric('meteor')
+
+    freeze_params(self.model.get_encoder())
+    self.freeze_embeds()
+  
+  def freeze_embeds(self):
+    ''' freeze the positional embedding parameters of the model; adapted from finetune.py '''
+    freeze_params(self.model.model.shared)
+    for d in [self.model.model.encoder, self.model.model.decoder]:
+      freeze_params(d.embed_positions)
+      freeze_params(d.embed_tokens)
+
+  # Do a forward pass through the model
+  def forward(self, input_ids, **kwargs):
+    return self.model(input_ids, **kwargs)
+  
+  def configure_optimizers(self):
+    optimizer = AdamW(self.parameters(), lr = self.learning_rate)
+    return optimizer
+
+  def shift_tokens_right(self, input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):
+    """
+    Shift input ids one token to the right.
+    https://github.com/huggingface/transformers/blob/main/src/transformers/models/bart/modeling_bart.py.
+    """
+    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
+    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
+    shifted_input_ids[:, 0] = decoder_start_token_id
+
+    if pad_token_id is None:
+        raise ValueError("self.model.config.pad_token_id has to be defined.")
+    # replace possible -100 values in labels by `pad_token_id`
+    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)
+
+    return shifted_input_ids
+
+  def run_model(self, batch):
+    src_ids, src_mask, tgt_ids = batch[0], batch[1], batch[2]
+
+    decoder_input_ids = self.shift_tokens_right(
+                tgt_ids, self.tokenizer.pad_token_id, self.tokenizer.pad_token_id # BART uses the EOS token to start generation as well. Might have to change for other models.
+            )
+
+    outputs = self(src_ids, attention_mask=src_mask, decoder_input_ids=decoder_input_ids, use_cache=False)
+    return outputs
+
+  def compute_loss(self, batch):
+    tgt_ids = batch[2]
+    logits = self.run_model(batch)[0]
+
+    cross_entropy = torch.nn.CrossEntropyLoss(ignore_index=self.tokenizer.pad_token_id)
+    loss = cross_entropy(logits.view(-1, logits.shape[-1]), tgt_ids.view(-1))
+
+    return loss
+
+  def training_step(self, batch, batch_idx):
+    loss = self.compute_loss(batch)
+
+    self.log("train_loss", loss, on_epoch=True)
+
+    return {'loss':loss}
+
+  def validation_step(self, batch, batch_idx):
+    preds, loss, tgts = self.generate_and_compute_loss_and_tgts(batch)
+    if self.should_pad_gen:
+      preds = F.pad(preds, pad=(0, self.gen_max_length - preds.shape[1]), value=self.tokenizer.pad_token_id)
+
+    self.log('val_loss', loss, prog_bar=True, sync_dist=True)
+
+    return {'loss': loss, 'pred': preds, 'target': tgts}
+
+  def test_step(self, batch, batch_idx):
+    test_preds, test_loss, test_tgts = self.generate_and_compute_loss_and_tgts(batch)
+    if self.should_pad_gen:
+      test_preds = F.pad(test_preds, pad=(0, self.gen_max_length - test_preds.shape[1]), value=self.tokenizer.pad_token_id)
+
+    self.log('test_loss', test_loss, prog_bar=True, sync_dist=True)
+
+    return {'loss': test_loss, 'pred': test_preds, 'target': test_tgts}
+
+  def test_epoch_end(self, outputs):
+    self.handle_end_of_epoch_scoring(outputs, "test")
+
+  def validation_epoch_end(self, outputs):
+    self.handle_end_of_epoch_scoring(outputs, "val")
+
+  def handle_end_of_epoch_scoring(self, outputs, prefix):
+      gen = {}
+      tgt = {}
+      rouge = ROUGEScore()
+      rouge_metric = lambda x, y: rouge(x,y)["rougeL_precision"]
+      for out in outputs:
+        preds = out['pred']
+        tgts = out['target']
+
+        preds = self.do_batch_detokenize(preds)
+        tgts = self.do_batch_detokenize(tgts)
+
+        for pred, t in zip(preds, tgts):
+          rouge_d = rouge_metric(pred, t)
+          self.log(prefix+"_rouge", rouge_d)
+
+          meteor_d = pairwise_meteor(pred, t)
+          self.log(prefix+"_meteor", meteor_d)
+
+  def generate_and_compute_loss_and_tgts(self, batch):
+    src_ids = batch[0]
+    loss = self.compute_loss(batch)
+    pred_ids, _ = self.generate_for_batch(src_ids)
+
+    tgt_ids = batch[2]
+
+    return pred_ids, loss, tgt_ids
+
+  def do_batch_detokenize(self, batch):
+    tokens = self.tokenizer.batch_decode(
+      batch, 
+      skip_special_tokens=True, 
+      clean_up_tokenization_spaces=True
+      )
+
+    # Huggingface skipping of special tokens doesn't work for all models, so we do it manually as well for safety:
+    tokens = [p.replace("<pad>", "") for p in tokens]
+    tokens = [p.replace("<s>", "") for p in tokens]
+    tokens = [p.replace("</s>", "") for p in tokens]
+
+    return [t for t in tokens if t != ""]
+  
+  def generate_for_batch(self, batch):
+    generated_ids = self.model.generate(
+      batch, 
+      decoder_start_token_id = self.tokenizer.pad_token_id,
+      num_beams = self.gen_num_beams,
+      max_length = self.gen_max_length
+      )
+
+    generated_tokens = self.tokenizer.batch_decode(
+      generated_ids, 
+      skip_special_tokens=True, 
+      clean_up_tokenization_spaces=True
+      )
+
+    return generated_ids, generated_tokens
+
+
+  def generate(self, text, max_input_length=512, device=None):
+    encoded_dict = self.tokenizer(
+            [text],
+            max_length=max_input_length,
+            padding="longest",
+            truncation=True,
+            return_tensors="pt",
+            add_prefix_space = True
+        )
+
+    input_ids = encoded_dict['input_ids']
+
+    if device is not None:
+      input_ids = input_ids.to(device)
+
+    with torch.no_grad():
+        _, generated_tokens = self.generate_for_batch(input_ids)
+    
+    return generated_tokens[0]

averitec/models/NaiveSeqClassModule.py

@@ -0,0 +1,145 @@
+import pytorch_lightning as pl
+import torch
+import numpy as np
+import datasets
+from transformers import MaxLengthCriteria, StoppingCriteriaList
+from transformers.optimization import AdamW
+import itertools
+from utils import count_stats, f1_metric, pairwise_meteor
+from torchmetrics.text.rouge import ROUGEScore
+import torch.nn.functional as F
+import torchmetrics
+from torchmetrics.classification import F1Score
+
+class NaiveSeqClassModule(pl.LightningModule):
+  # Instantiate the model
+  def __init__(self, tokenizer, model, use_question_stance_approach=True, learning_rate=1e-3):
+    super().__init__()
+    self.tokenizer = tokenizer
+    self.model = model
+    self.learning_rate = learning_rate
+
+    self.train_acc = torchmetrics.Accuracy()
+    self.val_acc = torchmetrics.Accuracy()
+    self.test_acc = torchmetrics.Accuracy()
+
+    self.train_f1 = F1Score(num_classes=4, average="macro")
+    self.val_f1 = F1Score(num_classes=4, average=None)
+    self.test_f1 = F1Score(num_classes=4, average=None)
+
+    self.use_question_stance_approach = use_question_stance_approach
+
+  
+  # Do a forward pass through the model
+  def forward(self, input_ids, **kwargs):
+    return self.model(input_ids, **kwargs)
+  
+  def configure_optimizers(self):
+    optimizer = AdamW(self.parameters(), lr = self.learning_rate)
+    return optimizer
+
+  def training_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+
+    outputs = self(x, attention_mask=x_mask, labels=y)
+    logits = outputs.logits
+    loss = outputs.loss
+
+    #cross_entropy = torch.nn.CrossEntropyLoss()
+    #loss = cross_entropy(logits, y)
+    
+    preds = torch.argmax(logits, axis=1)
+
+    self.train_acc(preds.cpu(), y.cpu())      
+    self.train_f1(preds.cpu(), y.cpu())      
+    
+    self.log("train_loss", loss)
+
+    return {'loss': loss}
+
+  def training_epoch_end(self, outs):
+    self.log('train_acc_epoch', self.train_acc)
+    self.log('train_f1_epoch', self.train_f1)
+
+  def validation_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+
+    outputs = self(x, attention_mask=x_mask, labels=y)
+    logits = outputs.logits
+    loss = outputs.loss
+
+    preds = torch.argmax(logits, axis=1)
+
+    if not self.use_question_stance_approach:
+      self.val_acc(preds, y)
+      self.log('val_acc_step', self.val_acc)
+
+      self.val_f1(preds, y)      
+      self.log("val_loss", loss)
+
+    return {'val_loss':loss, "src": x, "pred": preds, "target": y}
+
+  def validation_epoch_end(self, outs):
+    if self.use_question_stance_approach:
+      self.handle_end_of_epoch_scoring(outs, self.val_acc, self.val_f1)
+    
+    self.log('val_acc_epoch', self.val_acc)
+
+    f1 = self.val_f1.compute()
+    self.val_f1.reset()
+
+    self.log('val_f1_epoch', torch.mean(f1))
+
+    class_names = ["supported", "refuted", "nei", "conflicting"]
+    for i, c_name in enumerate(class_names):
+      self.log("val_f1_" + c_name, f1[i])
+
+
+  def test_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+    
+    outputs = self(x, attention_mask=x_mask)
+    logits = outputs.logits
+
+    preds = torch.argmax(logits, axis=1)
+    
+    if not self.use_question_stance_approach:
+      self.test_acc(preds, y)      
+      self.log('test_acc_step', self.test_acc)
+      self.test_f1(preds, y)      
+
+    return {"src": x, "pred": preds, "target": y}
+
+  def test_epoch_end(self, outs):
+    if self.use_question_stance_approach:
+      self.handle_end_of_epoch_scoring(outs, self.test_acc, self.test_f1)
+    
+    self.log('test_acc_epoch', self.test_acc)
+
+    f1 = self.test_f1.compute()
+    self.test_f1.reset()
+    self.log('test_f1_epoch', torch.mean(f1))
+
+    class_names = ["supported", "refuted", "nei", "conflicting"]
+    for i, c_name in enumerate(class_names):
+      self.log("test_f1_" + c_name, f1[i])
+
+  def handle_end_of_epoch_scoring(self, outputs, acc_scorer, f1_scorer):
+      gold_labels = {}
+      question_support = {}
+      for out in outputs:
+        srcs = out['src']
+        preds = out['pred']
+        tgts = out['target']
+
+        tokens = self.tokenizer.batch_decode(
+          srcs, 
+          skip_special_tokens=True, 
+          clean_up_tokenization_spaces=True
+        )
+
+        for src, pred, tgt in zip(tokens, preds, tgts):
+          acc_scorer(torch.as_tensor([pred]).to("cuda:0"), torch.as_tensor([tgt]).to("cuda:0"))    
+          f1_scorer(torch.as_tensor([pred]).to("cuda:0"), torch.as_tensor([tgt]).to("cuda:0"))
+
+        

averitec/models/SequenceClassificationModule.py

@@ -0,0 +1,179 @@
+import pytorch_lightning as pl
+import torch
+import numpy as np
+import datasets
+from transformers import MaxLengthCriteria, StoppingCriteriaList
+from transformers.optimization import AdamW
+import itertools
+# from utils import count_stats, f1_metric, pairwise_meteor
+from torchmetrics.text.rouge import ROUGEScore
+import torch.nn.functional as F
+import torchmetrics
+from torchmetrics.classification import F1Score
+
+class SequenceClassificationModule(pl.LightningModule):
+  # Instantiate the model
+  def __init__(self, tokenizer, model, use_question_stance_approach=True, learning_rate=1e-3):
+    super().__init__()
+    self.tokenizer = tokenizer
+    self.model = model
+    self.learning_rate = learning_rate
+
+    self.train_acc = torchmetrics.Accuracy(task="multiclass", num_classes=model.num_labels)
+    self.val_acc = torchmetrics.Accuracy(task="multiclass", num_classes=model.num_labels)
+    self.test_acc = torchmetrics.Accuracy(task="multiclass", num_classes=model.num_labels)
+
+    self.train_f1 = F1Score(task="multiclass", num_classes=model.num_labels, average="macro")
+    self.val_f1 = F1Score(task="multiclass", num_classes=model.num_labels, average=None)
+    self.test_f1 = F1Score(task="multiclass", num_classes=model.num_labels, average=None)
+    # self.train_acc = torchmetrics.Accuracy()
+    # self.val_acc = torchmetrics.Accuracy()
+    # self.test_acc = torchmetrics.Accuracy()
+
+    # self.train_f1 = F1Score(num_classes=4, average="macro")
+    # self.val_f1 = F1Score(num_classes=4, average=None)
+    # self.test_f1 = F1Score(num_classes=4, average=None)
+
+    self.use_question_stance_approach = use_question_stance_approach
+
+  
+  # Do a forward pass through the model
+  def forward(self, input_ids, **kwargs):
+    return self.model(input_ids, **kwargs)
+  
+  def configure_optimizers(self):
+    optimizer = AdamW(self.parameters(), lr = self.learning_rate)
+    return optimizer
+
+  def training_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+
+    outputs = self(x, attention_mask=x_mask, labels=y)
+    logits = outputs.logits
+    loss = outputs.loss
+
+    #cross_entropy = torch.nn.CrossEntropyLoss()
+    #loss = cross_entropy(logits, y)
+    
+    preds = torch.argmax(logits, axis=1)
+
+    self.log("train_loss", loss)
+
+    return {'loss': loss}
+
+  def validation_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+
+    outputs = self(x, attention_mask=x_mask, labels=y)
+    logits = outputs.logits
+    loss = outputs.loss
+
+    preds = torch.argmax(logits, axis=1)
+
+    if not self.use_question_stance_approach:
+      self.val_acc(preds, y)
+      self.log('val_acc_step', self.val_acc)
+
+      self.val_f1(preds, y)      
+      self.log("val_loss", loss)
+
+    return {'val_loss':loss, "src": x, "pred": preds, "target": y}
+
+  def validation_epoch_end(self, outs):
+    if self.use_question_stance_approach:
+      self.handle_end_of_epoch_scoring(outs, self.val_acc, self.val_f1)
+    
+    self.log('val_acc_epoch', self.val_acc)
+
+    f1 = self.val_f1.compute()
+    self.val_f1.reset()
+
+    self.log('val_f1_epoch', torch.mean(f1))
+
+    class_names = ["supported", "refuted", "nei", "conflicting"]
+    for i, c_name in enumerate(class_names):
+      self.log("val_f1_" + c_name, f1[i])
+
+
+  def test_step(self, batch, batch_idx):
+    x, x_mask, y = batch
+    
+    outputs = self(x, attention_mask=x_mask)
+    logits = outputs.logits
+
+    preds = torch.argmax(logits, axis=1)
+    
+    if not self.use_question_stance_approach:
+      self.test_acc(preds, y)      
+      self.log('test_acc_step', self.test_acc)
+      self.test_f1(preds, y)      
+
+    return {"src": x, "pred": preds, "target": y}
+
+  def test_epoch_end(self, outs):
+    if self.use_question_stance_approach:
+      self.handle_end_of_epoch_scoring(outs, self.test_acc, self.test_f1)
+    
+    self.log('test_acc_epoch', self.test_acc)
+
+    f1 = self.test_f1.compute()
+    self.test_f1.reset()
+    self.log('test_f1_epoch', torch.mean(f1))
+
+    class_names = ["supported", "refuted", "nei", "conflicting"]
+    for i, c_name in enumerate(class_names):
+      self.log("test_f1_" + c_name, f1[i])
+
+  def handle_end_of_epoch_scoring(self, outputs, acc_scorer, f1_scorer):
+      gold_labels = {}
+      question_support = {}
+      for out in outputs:
+        srcs = out['src']
+        preds = out['pred']
+        tgts = out['target']
+
+        tokens = self.tokenizer.batch_decode(
+          srcs, 
+          skip_special_tokens=True, 
+          clean_up_tokenization_spaces=True
+        )
+
+        for src, pred, tgt in zip(tokens, preds, tgts):
+          claim_id = src.split("[ question ]")[0]
+
+          if claim_id not in gold_labels:
+            gold_labels[claim_id] = tgt
+            question_support[claim_id] = []
+
+          question_support[claim_id].append(pred)
+
+      for k,gold_label in gold_labels.items():
+        support = question_support[k]
+
+        has_unansw = False
+        has_true = False
+        has_false = False
+
+        for v in support:
+          if v == 0:
+            has_true = True
+          if v == 1:
+            has_false = True
+          if v == 2 or v == 3: # TODO very ugly hack -- we cant have different numbers of labels for train and test so we do this
+            has_unansw = True
+
+        if has_unansw:
+          answer = 2
+        elif has_true and not has_false:
+          answer = 0
+        elif has_false and not has_true:
+          answer = 1
+        elif has_true and has_false:
+          answer = 3
+
+
+        # TODO this is very hacky and wont work if the device is literally anything other than cuda:0
+        acc_scorer(torch.as_tensor([answer]).to("cuda:0"), torch.as_tensor([gold_label]).to("cuda:0"))    
+        f1_scorer(torch.as_tensor([answer]).to("cuda:0"), torch.as_tensor([gold_label]).to("cuda:0"))
+
+        

averitec/models/utils.py

@@ -0,0 +1,119 @@
+import os
+import nltk
+from nltk import word_tokenize
+import numpy as np
+from leven import levenshtein
+from sklearn.cluster import DBSCAN, dbscan
+
+def delete_if_exists(filepath):
+    if os.path.exists(filepath):
+        os.remove(filepath)
+
+def pairwise_meteor(candidate, reference): # Todo this is not thread safe, no idea how to make it so
+    return nltk.translate.meteor_score.single_meteor_score(word_tokenize(reference), word_tokenize(candidate))
+
+def count_stats(candidate_dict, reference_dict):
+    count_match = [0 for _ in candidate_dict]
+    count_diff = [0 for _ in candidate_dict]
+
+    for i, k in enumerate(candidate_dict.keys()):
+      pred_parts = candidate_dict[k]
+      tgt_parts = reference_dict[k]
+
+      if len(pred_parts) == len(tgt_parts):
+        count_match[i] = 1
+
+      count_diff[i] = abs(len(pred_parts) - len(tgt_parts))
+
+    count_match_score = np.mean(count_match)
+    count_diff_score = np.mean(count_diff)
+
+    return {
+        "count_match_score": count_match_score,
+        "count_diff_score": count_diff_score
+    }
+
+def f1_metric(candidate_dict, reference_dict, pairwise_metric):
+    all_best_p = [0 for _ in candidate_dict]
+    all_best_t = [0 for _ in candidate_dict]
+    p_unnorm = []
+
+    for i, k in enumerate(candidate_dict.keys()):
+      pred_parts = candidate_dict[k]
+      tgt_parts = reference_dict[k]
+
+      best_p_score = [0 for _ in pred_parts]
+      best_t_score = [0 for _ in tgt_parts]
+
+      for p_idx in range(len(pred_parts)):
+        for t_idx in range(len(tgt_parts)):
+          #meteor_score = pairwise_meteor(pred_parts[p_idx], tgt_parts[t_idx])
+          metric_score = pairwise_metric(pred_parts[p_idx], tgt_parts[t_idx])
+
+          if metric_score > best_p_score[p_idx]:
+            best_p_score[p_idx] = metric_score
+
+          if metric_score > best_t_score[t_idx]:
+            best_t_score[t_idx] = metric_score
+
+      all_best_p[i] = np.mean(best_p_score) if len(best_p_score) > 0 else 1.0
+      all_best_t[i] = np.mean(best_t_score) if len(best_t_score) > 0 else 1.0       
+
+      p_unnorm.extend(best_p_score) 
+
+    p_score = np.mean(all_best_p)
+    r_score = np.mean(all_best_t)
+    avg_score = (p_score + r_score) / 2
+    f1_score = 2 * p_score * r_score / (p_score + r_score + 1e-8)
+
+    p_unnorm_score = np.mean(p_unnorm)
+
+    return {
+        "p": p_score,
+        "r": r_score,
+        "avg": avg_score,
+        "f1": f1_score,
+        "p_unnorm": p_unnorm_score,
+    }
+
+def edit_distance_dbscan(data):
+  # Inspired by https://scikit-learn.org/stable/faq.html#how-do-i-deal-with-string-data-or-trees-graphs
+  def lev_metric(x, y):
+    i, j = int(x[0]), int(y[0])
+    return levenshtein(data[i], data[j])
+
+  X = np.arange(len(data)).reshape(-1, 1)
+
+  clustering = dbscan(X, metric=lev_metric, eps=20, min_samples=2, algorithm='brute')
+  return clustering
+
+def compute_all_pairwise_edit_distances(data):
+  X = np.empty((len(data), len(data)))
+
+  for i in range(len(data)):
+    for j in range(len(data)):
+      X[i][j] = levenshtein(data[i], data[j])
+
+  return X
+
+def compute_all_pairwise_scores(src_data, tgt_data, metric):
+  X = np.empty((len(src_data), len(tgt_data)))
+
+  for i in range(len(src_data)):
+    for j in range(len(tgt_data)):
+      X[i][j] = (metric(src_data[i], tgt_data[j]))
+
+  return X
+
+def compute_all_pairwise_meteor_scores(data):
+  X = np.empty((len(data), len(data)))
+
+  for i in range(len(data)):
+    for j in range(len(data)):
+      X[i][j] = (pairwise_meteor(data[i], data[j]) + pairwise_meteor(data[j], data[i])) / 2
+
+  return X
+
+def edit_distance_custom(data, X, eps=0.5, min_samples=3):
+  clustering = DBSCAN(metric="precomputed", eps=eps, min_samples=min_samples).fit(X)
+  return clustering.labels_

averitec/pretrained_models/bart_justifications_verdict-epoch=13-val_loss=2.03-val_meteor=0.28.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4b7bf02daaf10b3443f4f2cbe79c3c9f10c453dfdf818a4d14e44b2b4311cf4
+size 4876206567

averitec/pretrained_models/bert_dual_encoder.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fee6737f655f4f1dfb46cc1bb812b5eaf9a72cfc0b69d4e5c05cde27ea7b6051
+size 1314015751

averitec/pretrained_models/bert_veracity.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ddb8132a28ceff149904dd3ad3c3edd3e5f0c7de0169819207104a80e425c9a
+size 1314034311