File size: 1,428 Bytes
158b61b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 |
#!/usr/bin/env perl
#
# This file is part of moses. Its use is licensed under the GNU Lesser General
# Public License version 2.1 or, at your option, any later version.
use strict;
my $minChars = $ARGV[0];
my $maxChars = $ARGV[1];
my $inputStem = $ARGV[2];
my $source = $ARGV[3];
my $target = $ARGV[4];
my $outputStem = $ARGV[5];
my $linesRetained = $ARGV[6];
open(IN_SOURCE, "<:encoding(UTF-8)", "$inputStem.$source") or die "cannot open $inputStem.$source";
open(IN_TARGET, "<:encoding(UTF-8)", "$inputStem.$target") or die "cannot open $inputStem.$target";
open(OUT_SOURCE, ">:encoding(UTF-8)", "$outputStem.$source") or die "cannot open $outputStem.$source";
open(OUT_TARGET, ">:encoding(UTF-8)", "$outputStem.$target") or die "cannot open $outputStem.$target";
open(LINE_RETAINED, ">:encoding(UTF-8)", "$linesRetained");
my $lineNum = 0;
while (my $lineSource = <IN_SOURCE>) {
++$lineNum;
#print STDERR "$lineNum ";
chomp($lineSource);
my $lineTarget = <IN_TARGET>;
chomp($lineTarget);
my $lenSource = length($lineSource);
my $lenTarget = length($lineTarget);
if ($lenSource < $minChars || $lenSource > $maxChars
|| $lenTarget < $minChars || $lenTarget > $maxChars) {
# do nothing
}
else {
print OUT_SOURCE "$lineSource\n";
print OUT_TARGET "$lineTarget\n";
print LINE_RETAINED "$lineNum\n";
}
}
close(OUT_SOURCE);
close(OUT_SOURCE);
close(LINE_RETAINED);
|