|
|
|
|
|
|
|
|
|
|
|
|
|
import gzip |
|
import logging |
|
from numpy import array |
|
import optparse |
|
import os.path |
|
import sys |
|
|
|
from nbest import * |
|
from sampler import * |
|
from train import * |
|
|
|
|
|
logging.basicConfig(format = "%(asctime)-15s %(message)s") |
|
log = logging.getLogger('main') |
|
log.setLevel(logging.DEBUG) |
|
|
|
class Config: |
|
def __init__(self): |
|
self.parser = optparse.OptionParser(usage="%prog [options] ") |
|
self.parser.add_option("-t", "--trainer", action="store",\ |
|
dest="trainer", metavar="TYPE", type="choice", choices=("pro","mix"),\ |
|
default="pro",\ |
|
help="type of trainer to run (pro,mix)") |
|
self.parser.add_option("-n", "--nbest", action="append", \ |
|
dest="nbest", metavar="NBEST-FILE",\ |
|
help="nbest output file(s) from decoder") |
|
self.parser.add_option("-S", "--scfile", action="append",\ |
|
dest="score", metavar="SCORE-FILE",\ |
|
help="score file(s) from extractor (in same order as nbests)") |
|
self.parser.add_option("-p", "--phrase-table" , action="append",\ |
|
dest="ttable", metavar="TTABLE",\ |
|
help="ttable to be used in mixture model training") |
|
self.parser.add_option("-i", "--input-file", action="store",\ |
|
dest="input_file", metavar="INPUT-FILE", |
|
help="source text file") |
|
self.parser.add_option("-m", "--moses-bin-dir", action="store",\ |
|
dest="moses_bin_dir", metavar="DIR", |
|
help="directory containing Moses binaries", |
|
default=os.path.expanduser("~/moses/bin")) |
|
self.nbest_files = [] |
|
self.score_files = [] |
|
self.ttables = [] |
|
|
|
def parse(self,args=sys.argv[1:]): |
|
(options,args) = self.parser.parse_args(args) |
|
self.nbest_files = options.nbest |
|
self.score_files = options.score |
|
self.ttables = options.ttable |
|
self.input_file = options.input_file |
|
self.trainer = options.trainer |
|
self.moses_bin_dir = options.moses_bin_dir |
|
if not self.nbest_files: |
|
self.nbest_files = ["data/esen.nc.nbest.segment"] |
|
if not self.score_files: |
|
self.score_files = ["data/esen.nc.scores"] |
|
if len(self.nbest_files) != len(self.score_files): |
|
self.parser.error("Must have equal numbers of score files and nbest files") |
|
if self.trainer == "mix": |
|
if not self.input_file or not self.ttables: |
|
self.parser.error("Need to specify input file and ttables for mix training") |
|
|
|
|
|
|
|
def main(): |
|
config = Config() |
|
config.parse() |
|
|
|
samples = [] |
|
sampler = HopkinsMaySampler() |
|
nbests = 0 |
|
for nbest_file,score_data_file in zip(config.nbest_files,config.score_files): |
|
log.debug("nbest: " + nbest_file + "; score:" + score_data_file) |
|
segments = False |
|
if config.trainer == "mix": segments = True |
|
for nbest in get_scored_nbests(nbest_file, score_data_file, config.input_file, segments=segments): |
|
samples += sampler.sample(nbest) |
|
nbests += 1 |
|
log.debug("Samples loaded") |
|
trainer = None |
|
if config.trainer == "mix": |
|
|
|
scorer = MosesPhraseScorer(config.ttables) |
|
log.debug("Scoring samples...") |
|
for sample in samples: |
|
scorer.add_scores(sample.hyp1) |
|
scorer.add_scores(sample.hyp2) |
|
log.debug("...samples scored") |
|
trainer = MixtureModelTrainer(samples) |
|
elif config.trainer == "pro": |
|
trainer = ProTrainer(samples) |
|
else: assert(0) |
|
log.debug("Starting training...") |
|
weights,mix_weights = trainer.train(debug=False) |
|
log.debug("...training complete") |
|
for i,w in enumerate(weights): |
|
print "F%d %10.8f" % (i,w) |
|
for i,f in enumerate(mix_weights): |
|
for j,w in enumerate(f): |
|
print "M%d_%d %10.8f" % (i,j,w) |
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
main() |
|
|
