File size: 5,331 Bytes
74e8f2f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 |
# Copyright 2024 Big Vision Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Evaluator for caption generation metrics used for the MS COCO dataset."""
import collections
import functools
import os
import tempfile
import big_vision.evaluators.common as c
import big_vision.input_pipeline
import big_vision.pp.builder
import big_vision.pp.tokenizer
import big_vision.utils as u
from pycocoevalcap.bleu import bleu
from pycocoevalcap.cider import cider
from pycocoevalcap.meteor import meteor
from pycocoevalcap.rouge import rouge
from pycocoevalcap.spice import spice
from pycocoevalcap.tokenizer import ptbtokenizer
import jax
from tensorflow.io import gfile
# Temporary global flag to facilitate backwards compatability. Will be removed
# by the end of year 2023.
API = "jit"
class Evaluator:
"""Evaluator for caption generation metrics used for the MS COCO dataset.
See https://arxiv.org/pdf/1504.00325.pdf or the repository implementing it
https://github.com/tylin/coco-caption for details on the metrics. This code
uses the python3 pip package from: https://github.com/salaniz/pycocoevalcap
Note that both the model caption and the ground truth reference captions are
further processed with the PTBTokenizer before computing scores.
`predict_fn` accepts arbitrary dictionaries of parameters and data, where
the data dictionary is produced by the `pp_fn` op. It is expected to output a
dict containing tokenized captions.
`pp_fn` must have fields: "image/id" and "captions".
"""
def __init__(
self, predict_fn, tokenizer=None,
metrics=("cider",), # Default to only cider. We often just look at that.
preds_outfile="{workdir}/{name}_{split}_preds.json",
annot_outfile="{workdir}/{name}_{split}_annotations.json",
*, data, devices, **kw
):
self.get_data_iter, self.steps = c.eval_input_pipeline(
keep_on_cpu={"image/id", "captions"}, data=data, devices=devices, **kw)
self.preds_outfile = c.resolve_outfile(
preds_outfile, name=data.get("name"), split=data.get("split"))
self.annot_outfile = c.resolve_outfile(
annot_outfile, name=data.get("name"), split=data.get("split"))
self.metrics = metrics
self.tok = big_vision.pp.tokenizer.get_tokenizer(tokenizer)
self.decode = functools.partial(
predict_fn, devices=devices, eos_token=self.tok.eos_token)
def run(self, train_state):
"""Run eval."""
gts = []
res = []
for _, batch in zip(range(self.steps), self.get_data_iter()):
# (batch, seqlen) array of decoded generated tokens.
tokens = self.decode(train_state, batch)
# (local_batch,)
tokens = u.get_local_slice_from_fsarray(tokens)
ex_masks = u.get_local_slice_from_fsarray(batch["_mask"])
image_ids = batch["image/id"][ex_masks]
pred_captions = self.tok.to_str(tokens[ex_masks])
for image_id, caption in zip(image_ids, pred_captions):
res.append({"image_id": image_id.item(), "caption": caption})
for image_id, captions in zip(image_ids, batch["captions"]):
for caption in captions:
gts.append({"image_id": image_id.item(), "caption": caption.item()})
# Write model outputs following: https://cocodataset.org/#format-results
# Use same format for gt although that is not the usual format for them.
res = c.multiprocess_write_json(self.preds_outfile, res)
gts = c.multiprocess_write_json(self.annot_outfile, gts)
if jax.process_index(): # Host0 gets all preds and does eval.
return
outs = self.evaluate(gts, res)
for key, score in outs.items():
yield key, score
def evaluate(self, gt_annotations, res_annotations):
"""Creates scorers and run evaluation."""
scorers = {
"rouge": rouge.Rouge,
"cider": cider.Cider,
"bleu-4": bleu.Bleu,
"spice": spice.Spice,
"meteor": meteor.Meteor,
}
# Reformat gts and res from [{"image_id": int|str, "caption": str}] to
# {int_image_id: [{"caption": str}]} as expected by tokenizer and scorers.
# Note there are multiple reference captions for the ground truth but only
# one for the model predictions.
iid_map = collections.defaultdict(lambda: len(iid_map))
res = {iid_map[x["image_id"]]: [x] for x in res_annotations}
gts = collections.defaultdict(list)
for x in gt_annotations:
gts[iid_map[x["image_id"]]].append(x)
assert sorted(gts.keys()) == sorted(res.keys())
# Tokenize captions and predictions using coco tokenizer.
coco_tokenizer = ptbtokenizer.PTBTokenizer()
gts = coco_tokenizer.tokenize(gts)
res = coco_tokenizer.tokenize(res)
scores = {}
for metric in self.metrics:
scorer = scorers[metric]()
scores[metric], _ = scorer.compute_score(gts, res)
return scores
|