|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"""Create a test corpus, using a previously pruned vocabulary.""" |
|
|
|
|
|
|
|
|
import logging |
|
|
import optparse |
|
|
import os |
|
|
import os.path |
|
|
import sys |
|
|
|
|
|
import extract |
|
|
|
|
|
|
|
|
def read_vocab(filename, offset=0): |
|
|
vocab = {} |
|
|
for i, line in enumerate(open(filename)): |
|
|
vocab[line.strip()] = i + offset |
|
|
return vocab, i + offset |
|
|
|
|
|
|
|
|
def main(): |
|
|
logging.basicConfig( |
|
|
format='%(asctime)s %(levelname)s: %(message)s', |
|
|
datefmt='%Y-%m-%d %H:%M:%S', level=logging.DEBUG) |
|
|
parser = optparse.OptionParser("%prog [options]") |
|
|
parser.add_option( |
|
|
"-e", "--target-language", type="string", dest="target_language") |
|
|
parser.add_option( |
|
|
"-f", "--source-language", type="string", dest="source_language") |
|
|
parser.add_option( |
|
|
"-c", "--corpus", type="string", dest="corpus_stem") |
|
|
parser.add_option( |
|
|
"-t", "--tagged-corpus", type="string", dest="tagged_stem") |
|
|
parser.add_option( |
|
|
"-a", "--align", type="string", dest="align_file") |
|
|
parser.add_option( |
|
|
"-w", "--working-dir", type="string", dest="working_dir") |
|
|
|
|
|
parser.set_defaults( |
|
|
target_language="en", |
|
|
source_language="de", |
|
|
corpus_stem="test", |
|
|
align_file="test.align", |
|
|
working_dir="working") |
|
|
options, args = parser.parse_args(sys.argv) |
|
|
if not os.path.exists(options.working_dir): |
|
|
raise Exception( |
|
|
"Working directory '%s' not found" % options.working_dir) |
|
|
|
|
|
m, n = None, None |
|
|
for line in open(options.working_dir + "/info"): |
|
|
name, value = line[:-1].split() |
|
|
if name == "m": |
|
|
m = int(value) |
|
|
if name == "n": |
|
|
n = int(value) |
|
|
if m is None or n is None: |
|
|
raise Exception("Info file is incomplete.") |
|
|
|
|
|
tvocab, offset = read_vocab(options.working_dir + "/vocab.target") |
|
|
svocab, offset = read_vocab( |
|
|
options.working_dir + "/vocab.source", offset + 1) |
|
|
|
|
|
file_stem = os.path.basename(options.corpus_stem) |
|
|
ofh = open(options.working_dir + "/" + file_stem + ".ngrams", "w") |
|
|
extract.get_ngrams( |
|
|
options.corpus_stem, |
|
|
options.align_file, |
|
|
options.tagged_stem, |
|
|
svocab, |
|
|
tvocab, |
|
|
options.source_language, |
|
|
options.target_language, |
|
|
m, |
|
|
n, |
|
|
ofh) |
|
|
|
|
|
numberized_file = options.working_dir + "/" + file_stem + ".numberized" |
|
|
ngrams_file_handle = open( |
|
|
os.path.join(options.working_dir, file_stem + ".ngrams"), 'r') |
|
|
numberized_file_handle = open(numberized_file, 'w') |
|
|
|
|
|
|
|
|
for line in ngrams_file_handle: |
|
|
numberized_file_handle.write(extract.numberize( |
|
|
line, m, n, svocab, tvocab)) |
|
|
|
|
|
numberized_file_handle.close() |
|
|
ngrams_file_handle.close() |
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
main() |
|
|
|
