File size: 1,106 Bytes
158b61b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
#!/usr/bin/env perl
#
# This file is part of moses. Its use is licensed under the GNU Lesser General
# Public License version 2.1 or, at your option, any later version.
use warnings;
use strict;
use FindBin qw($RealBin);
use Getopt::Long "GetOptions";
my ($IN,$OUT,$MXPOST);
if (!&GetOptions('mxpost=s' => \$MXPOST) ||
!($IN = shift @ARGV) ||
!($OUT = shift @ARGV) ||
!defined($MXPOST)) {
print "syntax: make-pos-en.mxpost.perl -mxpost INSTALL_DIR IN_FILE OUT_FILE\n";
exit(1);
}
my $pipeline = "perl -ne 'chop; tr/\\x20-\\x7f/\?/c; print \$_.\"\\n\";' | tee debug | ";
$pipeline .= "$MXPOST/mxpost $MXPOST/tagger.project |";
open(TAGGER,"$RealBin/../../tokenizer/deescape-special-chars.perl < $IN | $pipeline");
open(OUT,"| $RealBin/../../tokenizer/escape-special-chars.perl > $OUT");
while(<TAGGER>) {
foreach my $word_pos (split) {
$word_pos =~ s/\/([^\/]+)$/_$1/;
$word_pos = "//_:" if $word_pos eq "//";
print STDERR "faulty POS tag: $word_pos\n"
unless $word_pos =~ /^.+_([^_]+)$/;
print OUT "$1 ";
}
print OUT "\n";
}
close(OUT);
close(TAGGER);
|