File size: 2,218 Bytes
158b61b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 |
#!/usr/bin/env perl
#
# This file is part of moses. Its use is licensed under the GNU Lesser General
# Public License version 2.1 or, at your option, any later version.
use warnings;
use strict;
my $jobs = 20;
my ($infile,$outfile,$cmd,$tmpdir,$qflags);
use Getopt::Long qw(:config pass_through no_ignore_case);
GetOptions('jobs=i' => \$jobs,
'tmpdir=s' => \$tmpdir,
'in=s' => \$infile,
'out=s' => \$outfile,
'cmd=s' => \$cmd,
'queue-flags=s' => \$qflags,
) or exit(1);
die("ERROR: specify infile with -in") unless $infile;
die("ERROR: specify outfile with -out") unless $outfile;
die("ERROR: did not find infile '$infile'") unless -e $infile;
die("ERROR: you need to specify a tempdir with -tmpdir") unless $tmpdir;
$qflags = "" unless $qflags;
# set up directory
`mkdir -p $tmpdir`;
# create split input files
my $sentenceN = `cat $infile | wc -l`;
my $splitN = int(($sentenceN+$jobs-0.5) / $jobs);
`split -a 2 -l $splitN $infile $tmpdir/in-$$-`;
# find out the names of the jobs
my @JOB=`ls $tmpdir/in-$$-*`;
chomp(@JOB);
grep(s/.+in\-\d+\-([a-z]+)$/$1/e,@JOB);
# create job scripts
foreach my $job (@JOB){
open(BASH,">$tmpdir/job-$$-$job.bash") or die "Cannot open: $!";
print BASH "#bash\n\n";
# print BASH "export PATH=$ENV{PATH}\n\n";
printf BASH $cmd."\n", "$tmpdir/in-$$-$job", "$tmpdir/out-$$-$job";
close(BASH);
}
# submit jobs
foreach my $job (@JOB){
my $cmd = "qsub $qflags -b no -j yes "
."-o $tmpdir/job-$$-$job.stdout "
."-e $tmpdir/job-$$-$job.stderr "
."-N J$$-$job "
."$tmpdir/job-$$-$job.bash "
.">& $tmpdir/job-$$-$job.log";
print STDERR $cmd."\n";
print STDERR `$cmd`;
}
# get qsub ID
my @QSUB_ID;
foreach my $job (@JOB){
`cat $tmpdir/job-$$-$job.log` =~ /Your job (\d+) /
or die "ERROR: Can't read log of job $tmpdir/job-$$-$job.log";
push @QSUB_ID,$1;
}
# create hold job and wait
my $hj = "-hold_jid " . join(" -hold_jid ", @QSUB_ID);
my $qsubname = "hold-$$";
$cmd="qsub $qflags -sync y $hj -j y -o /dev/null -e /dev/null -N $qsubname.W -b y /bin/ls >& $tmpdir/$qsubname.W.log";
print $cmd."\n";
`$cmd`;
# merge outfile
`rm -rf $outfile`;
foreach my $job (@JOB){
`cat $tmpdir/out-$$-$job >> $outfile`;
}
|