File size: 4,812 Bytes
74e8f2f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 |
# Copyright 2024 Big Vision Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Evaluator for the POPE dataset (https://github.com/RUCAIBox/POPE).
POPE is a binary classification dataset with ground-truth answers being either
'yes' or 'no'.
"""
import functools
import big_vision.datasets.core
import big_vision.evaluators.common as c
import big_vision.input_pipeline
import big_vision.pp.builder
import big_vision.pp.tokenizer
import big_vision.utils as u
# Temporary global flag to facilitate backwards compatability. Will be removed
# by the end of year 2023.
API = "jit"
class Evaluator:
"""Evaluator for the POPE task.
This evaluator expects the batch to contain a field `question_id` and a field
`answer` for single ground truth or `answers` for multiple ground truths.
The field names used when writting the json result can be controlled with
`out_question_key` and `out_answer_key`.
"""
def __init__(
self,
predict_fn,
data,
pp_fn,
tokenizer,
batch_size,
*,
devices,
outfile="{workdir}/{split}.json",
out_question_key="question_id",
out_answer_key="answer"
):
self.outfile = c.resolve_outfile(outfile, split=data.get("split"))
self.out_question_key = out_question_key
self.out_answer_key = out_answer_key
# This will mostly look the same across all evaluators, preparing data:
data = big_vision.datasets.core.get(**data)
pp_fn = big_vision.pp.builder.get_preprocess_fn(pp_fn)
self.ds, self.steps = big_vision.input_pipeline.make_for_inference(
data.get_tfdata(ordered=True),
pp_fn,
batch_size,
num_ex_per_process=data.num_examples_per_process(),
)
# The `keep_on_cpu=` argument lists the data keys that, if they exist, we
# do NOT want to ship to the TPUs and instead just keep in host memory.
# Typically ground-truth and metadata, that is often of string type.
self.data_iter = big_vision.input_pipeline.start_global(
self.ds, devices, keep_on_cpu={"answer", "question_id"}
)
# We'll need the tokenizer to detokenize the model outputs later.
self.tok = big_vision.pp.tokenizer.get_tokenizer(tokenizer)
self.decode = functools.partial(
predict_fn, devices=devices, eos_token=self.tok.eos_token
)
def run(self, train_state):
"""Does one evaluation run, yields metrics."""
accuracies = []
valid = []
json_out = []
for _, batch in zip(range(self.steps), self.data_iter):
# (batch, seqlen) array of decoded generated tokens.
tokens = self.decode(train_state, batch)
# (local_batch,) that indicates padding examples (0) vs real examples (1).
tokens = u.get_local_slice_from_fsarray(tokens)
ex_masks = u.get_local_slice_from_fsarray(batch["_mask"])
# Turn predictions into texts and then scores, one by one.
for i in range(len(tokens)):
if ex_masks[i] == 0: # Skip last-batch padding examples
continue
answer = self.tok.to_str(tokens[i], stop_at_eos=True).lower()
gt = batch["answer"][i]
accuracies.append(float(answer == gt))
valid.append(float(answer in ("yes", "no")))
json_out.append(
{
self.out_question_key: batch["question_id"][i].item(),
self.out_answer_key: answer,
}
)
# At this point `accuracies` is a list of per-example scores. However,
# remember that each host holds a different subset of the examples! So if
# we were to just return the mean accuracy here, we would effectively only
# have evaluated on the main host's (who writes metrics) subset!
# So now, we need to compute global means.
# There is one more caveat: `process_sum` needs the summands on each host
# to have the same size. So we either need to include dummy values for
# the padding examples (last batch, annoying), or we only sum scalars as in
# sufficient statistics, which we do here.
sum_accs, sum_valid, num = c.process_sum([
sum(accuracies),
sum(valid),
len(accuracies),
])
if num:
yield "acc", sum_accs / num
yield "valid_percent", sum_valid / num
yield "num", num
c.multiprocess_write_json(self.outfile, json_out)
|