Spaces:
Runtime error
Runtime error
File size: 4,900 Bytes
8044721 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 |
# Marco Kuhlmann <[email protected]>
import sys
from score.core import anchor, intersect;
class Measure(object):
def __init__(self, get_items):
self.get_items = get_items
self.g = 0
self.s = 0
self.c = 0
self.n_updates = 0
self.n_matches = 0
def update(self, gold, system, gidentities, sidentities, trace = 0):
g_items = set(self.get_items(gold, gidentities))
s_items = set(self.get_items(system, sidentities))
self.g += len(g_items)
self.s += len(s_items)
self.c += len(g_items & s_items)
self.n_updates += 1
self.n_matches += g_items == s_items
if trace:
return {"g": len(g_items), "s": len(s_items),
"c": len(g_items & s_items), "m": 1 if g_items == s_items else 0};
def p(self):
return self.c / self.s if self.s != 0 else 0.0
def r(self):
return self.c / self.g if self.g != 0 else 0.0
def f(self):
p = self.p()
r = self.r()
return 2 * p * r / (p + r) if p + r != 0 else 0.0
def m(self):
return self.n_matches / self.n_updates if self.n_updates != 0 else 0.0
def report(self):
json = {}
json["g"] = self.g
json["s"] = self.s
json["c"] = self.c
json["p"] = self.p()
json["r"] = self.r()
json["f"] = self.f()
json["m"] = self.m()
return json
# def argument_predicate_dm(label):
# return True
# def argument_predicate_pas(label):
# arguments = set("adj_ARG1 adj_ARG2 adj_MOD coord_ARG1 coord_ARG2 prep_ARG1 prep_ARG2 prep_ARG3 prep_MOD verb_ARG1 verb_ARG2 verb_ARG3 verb_ARG4 verb_MOD".split())
# return label in arguments
# def argument_predicate_psd(label):
# return label.endswith("-arg")
class Scorer(object):
def __init__(self, include_virtual=True):
self.measures = []
self.measures.append(("labeled", Measure(self.get_itemsL)))
self.measures.append(("unlabeled", Measure(self.get_itemsU)))
# self.measureP = Measure(self.get_itemsP)
# self.measureF = Measure(self.get_itemsF)
# self.measureS = Measure(self.get_itemsS)
self.include_virtual = include_virtual
def identify(self, id):
return self.identities[id]
def get_itemsL(self, graph, identities):
result = {(identities[e.src], identities[e.tgt], e.lab) for e in graph.edges}
if self.include_virtual:
for node in graph.nodes:
if node.is_top:
result.add((-1, identities[node.id], None))
return result
def get_itemsU(self, graph, identities):
result = {(identities[e.src], identities[e.tgt]) for e in graph.edges}
if self.include_virtual:
for node in graph.nodes:
if node.is_top:
result.add((-1, identities[node.id]))
return result
# def get_itemsP(self, graph):
# return {(frame[0], frame[2]) for frame in self.get_itemsF(graph)}
# def get_itemsF(self, graph):
# result = set()
# for node in graph.nodes:
# if self.has_scorable_predicate(node):
# arguments = set()
# for edge in node.outgoing_edges:
# if self.argument_predicate(edge.lab):
# arguments.add(edge)
# extract = (node.id, node.sense, tuple(sorted(arguments)))
# result.add(extract)
# return result
# def get_itemsS(self, graph):
# return {(frame[0], frame[1]) for frame in self.get_itemsF(graph)}
# def argument_predicate(self, label):
# return True
# def has_scorable_predicate(self, node):
# return node.pred and node.pos.startswith("V")
# def show_predications(self, g):
# print(g.id)
# report_predications(self.complete_predications(g))
def update(self, g, s, trace):
gidentities = {node.id: tuple(anchor(node)) for node in g.nodes}
sidentities = {node.id: tuple(anchor(node)) for node in s.nodes}
scores = dict();
for key, measure in self.measures:
score = measure.update(g, s, gidentities, sidentities, trace)
if trace: scores[key] = score;
return scores;
def report(self, n, scores = None):
json = {"n": n}
for info, measure in self.measures:
json[info] = measure.report()
if scores is not None: json["scores"] = scores
return json
def evaluate(gold, system, format = "json", trace = 0):
scorer = Scorer(include_virtual=True)
n = 0
scores = dict() if trace else None
for g, s in intersect(gold, system):
score = scorer.update(g, s, trace)
n += 1
if trace: scores[g.id] = score
result = scorer.report(n, scores)
return result
|