|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
from argparse import ArgumentParser |
|
|
import math |
|
|
import os |
|
|
from random import randint |
|
|
import sys, gzip |
|
|
|
|
|
|
|
|
def count_ngrams(snt, max_n): |
|
|
""" |
|
|
Return a dictionary of ngram counts (up to length /max_n/) |
|
|
for sentence (list of words) /snt/. |
|
|
""" |
|
|
ret = {} |
|
|
for i in xrange(len(snt)): |
|
|
for k in xrange(i + 1, min(i + max_n + 1, len(snt) + 1)): |
|
|
key = tuple(snt[i:k]) |
|
|
ret[key] = ret.get(key, 0) + 1 |
|
|
return ret |
|
|
|
|
|
|
|
|
def max_counts(ng1, ng2): |
|
|
""" |
|
|
Return a dicitonary of ngram counts such that |
|
|
each count is the greater of the two individual counts |
|
|
for each ngram in the input ngram count dictionaries |
|
|
/ng1/ and /ng2/. |
|
|
""" |
|
|
ret = ng1.copy() |
|
|
for k, v in ng2.items(): |
|
|
ret[k] = max(ret.get(k, 0), v) |
|
|
return ret |
|
|
|
|
|
|
|
|
def ng_hits(hyp, ref, max_n): |
|
|
""" |
|
|
Return a list of ngram counts such that each ngram count |
|
|
is the minimum of the counts in hyp and ref, up to ngram |
|
|
length /max_n/. |
|
|
""" |
|
|
ret = [0 for i in xrange(max_n)] |
|
|
for ng, cnt in hyp.items(): |
|
|
k = ng |
|
|
if len(k) <= max_n: |
|
|
ret[len(k) - 1] += min(cnt, ref.get(ng, 0)) |
|
|
return ret |
|
|
|
|
|
|
|
|
class BleuScore: |
|
|
def __init__(self, hyp, ref, max_n=4, bootstrap=1000): |
|
|
|
|
|
self.hits = [ |
|
|
ng_hits(hyp.ngrams[i], ref.ngrams[i], max_n) |
|
|
for i in xrange(len(hyp.ngrams))] |
|
|
self.max_n = max_n |
|
|
self.hyp = hyp |
|
|
self.ref = ref |
|
|
self.lower = None |
|
|
self.upper = None |
|
|
self.median = None |
|
|
self.actual = self.score([i for i in xrange(len(hyp.snt))]) |
|
|
if bootstrap: |
|
|
self.bootstrap = [self.score([randint(0, len(hyp.snt) - 1) |
|
|
for s in hyp.snt]) |
|
|
for i in xrange(bootstrap)] |
|
|
self.bootstrap.sort() |
|
|
else: |
|
|
self.bootstrap = [self.actual] |
|
|
pass |
|
|
|
|
|
def score(self, sample): |
|
|
hits = [0 for i in xrange(self.max_n)] |
|
|
self.hyplen = 0 |
|
|
self.reflen = 0 |
|
|
self.total = [0 for i in hits] |
|
|
for i in sample: |
|
|
self.hyplen += len(self.hyp.snt[i]) |
|
|
self.reflen += len(self.ref.snt[i]) |
|
|
for n in xrange(self.max_n): |
|
|
hits[n] += self.hits[i][n] |
|
|
self.total[n] += max(len(self.hyp.snt[i]) - n, 0) |
|
|
pass |
|
|
self.prec = [float(hits[n]) / self.total[n] |
|
|
for n in xrange(self.max_n)] |
|
|
ret = sum([math.log(x) for x in self.prec]) / self.max_n |
|
|
self.BP = min( |
|
|
1, math.exp(1. - float(self.reflen) / float(self.hyplen))) |
|
|
ret += math.log(self.BP) |
|
|
return math.exp(ret) |
|
|
|
|
|
|
|
|
class Document: |
|
|
def __init__(self, fname=None): |
|
|
self.fname = fname |
|
|
if fname: |
|
|
if fname[-3:] == ".gz": |
|
|
self.snt = [line.strip().split() for line in gzip.open(fname).readlines()] |
|
|
else: |
|
|
self.snt = [line.strip().split() for line in open(fname)] |
|
|
pass |
|
|
self.ngrams = [count_ngrams(snt, 4) for snt in self.snt] |
|
|
|
|
|
else: |
|
|
self.snt = None |
|
|
self.ngrams = None |
|
|
|
|
|
def merge(self, R): |
|
|
self.fname = "multi-ref" |
|
|
self.ngrams = [x for x in R[0].ngrams] |
|
|
self.snt = [x for x in R[0].snt] |
|
|
for i in xrange(len(R[0].ngrams)): |
|
|
for k in xrange(1, len(R)): |
|
|
self.ngrams[i] = max_counts(self.ngrams[i], R[k].ngrams[i]) |
|
|
|
|
|
def update(self, hyp, R): |
|
|
for i, hyp_snt in enumerate(hyp.snt): |
|
|
clen = len(hyp_snt) |
|
|
K = 0 |
|
|
for k in xrange(1, len(R)): |
|
|
k_snt = R[k].snt[i] |
|
|
assert len(R[k].snt) == len(hyp.snt), ( |
|
|
"Mismatch in number of sentences " + |
|
|
"between reference and candidate") |
|
|
if abs(len(k_snt) - clen) == abs(len(R[K].snt[i]) - clen): |
|
|
if len(k_snt) < len(R[K].snt[i]): |
|
|
K = k |
|
|
elif abs(len(k_snt) - clen) < abs(len(R[K].snt[i]) - clen): |
|
|
K = k |
|
|
self.snt[i] = R[K].snt[i] |
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
argparser = ArgumentParser() |
|
|
argparser.add_argument( |
|
|
"-r", "--ref", nargs='+', help="Reference translation(s).") |
|
|
argparser.add_argument( |
|
|
"-c", "--cand", nargs='+', help="Candidate translations.") |
|
|
argparser.add_argument( |
|
|
"-i", "--individual", action='store_true', |
|
|
help="Compute BLEU scores for individual references.") |
|
|
argparser.add_argument( |
|
|
"-b", "--bootstrap", type=int, default=1000, |
|
|
help="Sample size for bootstrap resampling.") |
|
|
argparser.add_argument( |
|
|
"-a", "--alpha", type=float, default=.05, |
|
|
help="1-alpha = confidence interval.") |
|
|
args = argparser.parse_args(sys.argv[1:]) |
|
|
R = [Document(fname) for fname in args.ref] |
|
|
C = [Document(fname) for fname in args.cand] |
|
|
Rx = Document() |
|
|
Rx.merge(R) |
|
|
for c in C: |
|
|
|
|
|
Rx.update(c, R) |
|
|
bleu = BleuScore(c, Rx, bootstrap=args.bootstrap) |
|
|
print "%5.2f %s [%5.2f-%5.2f; %5.2f] %s" % ( |
|
|
100 * bleu.actual, |
|
|
os.path.basename(Rx.fname), |
|
|
100 * bleu.bootstrap[int((args.alpha / 2) * args.bootstrap)], |
|
|
100 * bleu.bootstrap[int((1 - (args.alpha / 2)) * args.bootstrap)], |
|
|
100 * bleu.bootstrap[int(.5 * args.bootstrap)], |
|
|
c.fname) |
|
|
|
|
|
if args.individual: |
|
|
for r in R: |
|
|
bleu = BleuScore(c, r, bootstrap=args.bootstrap) |
|
|
print " %5.2f %s" % ( |
|
|
100 * bleu.actual, os.path.basename(r.fname)) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
