File size: 5,408 Bytes
158b61b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 |
# This file is part of moses. Its use is licensed under the GNU Lesser General
# Public License version 2.1 or, at your option, any later version.
package LexicalTranslationModel;
use strict;
use warnings;
BEGIN {
require Exporter;
our $VERSION = 1.0;
our @ISA = qw(Exporter);
our @EXPORT = qw(get_lexical);
our @EXPORT_OK = qw();
}
sub open_compressed {
# utilities
my $ZCAT = "gzip -cd";
my $BZCAT = "bzcat";
my ($file) = @_;
print "FILE: $file\n";
# add extensions, if necessary
$file = $file.".bz2" if ! -e $file && -e $file.".bz2";
$file = $file.".gz" if ! -e $file && -e $file.".gz";
# pipe zipped, if necessary
return "$BZCAT $file|" if $file =~ /\.bz2$/;
return "$ZCAT $file|" if $file =~ /\.gz$/;
return $file;
}
sub fix_spaces {
my ($in) = @_;
$$in =~ s/[ \t]+/ /g; $$in =~ s/[ \t]$//; $$in =~ s/^[ \t]//;
}
sub get_lexical {
my ($alignment_file_f,$alignment_file_e,$alignment_file_a,$lexical_file,$write_counts,$baseline_corpus_f,$baseline_corpus_e,$baseline_alignment, $instance_weights_file) = @_;
print STDERR "($alignment_file_f,$alignment_file_e,$lexical_file)\n";
print STDERR "baseline ($baseline_corpus_f,$baseline_corpus_e,$baseline_alignment)\n" if defined $baseline_alignment;
print STDERR "instance weights ($instance_weights_file)\n" if defined $instance_weights_file;
# my $alignment_file_a = $___ALIGNMENT_FILE.".".$___ALIGNMENT;
if (-e "$lexical_file.f2e" && -e "$lexical_file.e2f" && (!$write_counts || -e "$lexical_file.counts.f2e" && -e "$lexical_file.counts.e2f")) {
print STDERR " reusing: $lexical_file.f2e and $lexical_file.e2f\n";
return;
}
my (%WORD_TRANSLATION,%TOTAL_FOREIGN,%TOTAL_ENGLISH);
&get_lexical_counts($alignment_file_e,$alignment_file_f,$alignment_file_a,$instance_weights_file,\%WORD_TRANSLATION,\%TOTAL_FOREIGN,\%TOTAL_ENGLISH);
if (defined($baseline_alignment)) {
&get_lexical_counts($baseline_corpus_e,$baseline_corpus_f,$baseline_alignment,undef,\%WORD_TRANSLATION,\%TOTAL_FOREIGN,\%TOTAL_ENGLISH);
}
open(F2E,">$lexical_file.f2e") or die "ERROR: Can't write $lexical_file.f2e";
open(E2F,">$lexical_file.e2f") or die "ERROR: Can't write $lexical_file.e2f";
if ($write_counts) {
open(F2E2,">$lexical_file.counts.f2e") or die "ERROR: Can't write $lexical_file.counts.f2e";
open(E2F2,">$lexical_file.counts.e2f") or die "ERROR: Can't write $lexical_file.counts.e2f";
}
foreach my $f (keys %WORD_TRANSLATION) {
foreach my $e (keys %{$WORD_TRANSLATION{$f}}) {
printf F2E "%s %s %.7f\n",$e,$f,$WORD_TRANSLATION{$f}{$e}/$TOTAL_FOREIGN{$f};
printf E2F "%s %s %.7f\n",$f,$e,$WORD_TRANSLATION{$f}{$e}/$TOTAL_ENGLISH{$e};
if ($write_counts) {
printf F2E2 "%s %s %i %i\n",$e,$f,$WORD_TRANSLATION{$f}{$e},$TOTAL_FOREIGN{$f};
printf E2F2 "%s %s %i %i\n",$f,$e,$WORD_TRANSLATION{$f}{$e},$TOTAL_ENGLISH{$e};
}
}
}
close(E2F);
close(F2E);
if ($write_counts) {
close(E2F2);
close(F2E2);
}
print STDERR "Saved: $lexical_file.f2e and $lexical_file.e2f\n";
}
sub get_lexical_counts {
my ($alignment_file_e,$alignment_file_f,$alignment_file_a,$instance_weights_file,$WORD_TRANSLATION,$TOTAL_FOREIGN,$TOTAL_ENGLISH) = @_;
open(E,&open_compressed($alignment_file_e)) or die "ERROR: Can't read $alignment_file_e";
open(F,&open_compressed($alignment_file_f)) or die "ERROR: Can't read $alignment_file_f";
open(A,&open_compressed($alignment_file_a)) or die "ERROR: Can't read $alignment_file_a";
my $W = undef;
if (defined($instance_weights_file) && $instance_weights_file) {
open($W, $instance_weights_file) or die "ERROR: Can't read $instance_weights_file";
}
my $alignment_id = 0;
while(my $e = <E>) {
if (($alignment_id++ % 1000) == 0) { print STDERR "!"; }
chomp($e); fix_spaces(\$e);
my @ENGLISH = split(/ /,$e);
my $f = <F>; chomp($f); fix_spaces(\$f);
my @FOREIGN = split(/ /,$f);
my $a = <A>; chomp($a); fix_spaces(\$a);
my $iw = 1; # instance weight
$iw = <$W> if defined $W;
my (%FOREIGN_ALIGNED,%ENGLISH_ALIGNED);
foreach (split(/ /,$a)) {
my ($fi,$ei) = split(/\-/);
if ($fi >= scalar(@FOREIGN) || $ei >= scalar(@ENGLISH)) {
print STDERR "alignment point ($fi,$ei) out of range (0-$#FOREIGN,0-$#ENGLISH) in line $alignment_id, ignoring\n";
}
else {
# local counts
$FOREIGN_ALIGNED{$fi}+=$iw;
$ENGLISH_ALIGNED{$ei}+=$iw;
# global counts
$$WORD_TRANSLATION{$FOREIGN[$fi]}{$ENGLISH[$ei]}+=$iw;
$$TOTAL_FOREIGN{$FOREIGN[$fi]}+=$iw;
$$TOTAL_ENGLISH{$ENGLISH[$ei]}+=$iw;
}
}
# unaligned words
for(my $ei=0;$ei<scalar(@ENGLISH);$ei++) {
next if defined($ENGLISH_ALIGNED{$ei});
$$WORD_TRANSLATION{"NULL"}{$ENGLISH[$ei]}+=$iw;
$$TOTAL_ENGLISH{$ENGLISH[$ei]}+=$iw;
$$TOTAL_FOREIGN{"NULL"}+=$iw;
}
for(my $fi=0;$fi<scalar(@FOREIGN);$fi++) {
next if defined($FOREIGN_ALIGNED{$fi});
$$WORD_TRANSLATION{$FOREIGN[$fi]}{"NULL"}+=$iw;
$$TOTAL_FOREIGN{$FOREIGN[$fi]}+=$iw;
$$TOTAL_ENGLISH{"NULL"}+=$iw;
}
}
print STDERR "\n";
close(A);
close(F);
close(E);
}
END {
}
1;
|