File size: 2,507 Bytes
158b61b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
#!/bin/bash
# Helper script for phrase extraction from a single corpus shard.
# Written by Ulrich Germann.
# to be added: built-in factor filtering for factored models
cleanup()
{
if [ -e $fifo ] ; then rm $fifo; fi
if [ -e $fifo.inv ] ; then rm $fifo.inv; fi
if [ -e $fifo.o ] ; then rm $fifo.o; fi
}
usage()
{
echo
echo "$0: wrapper script to extract phrases from word-aligned corpus"
echo -e "usage:\n $0 <extractor> <ibase> <L1tag> <L2tag> [-x] "
echo "options:"
echo "-l: maximum phrase length ($plen)"
echo "-m: distortion model specification"
echo "-o: base name for output files .fwd.gz .bwd.gz [.<dmodel>.dst.gz]"
echo "-x: (no argument) don't create .fwd.gz and .bwd.gz"
echo
echo "required input files: <ibase>.<L1tag>.gz ibase.<L2tag>.gz ibase.<aln>.gz"
}
plen=7
nottable=
dmodel=
dspec=
pargs=
sfactors=
tfactors=
while [ $# -gt 0 ]; do
case $1 in
-l*) plen=${1#-l}
plen=${plen#=}
if [ -z $plen ] ; then
shift
plen=$1
fi
;;
-m*) dmodel=${1#-m}
dmodel=${dmodel#=}
if [ -z $dmodel ] ; then
shift
dmodel="$1"
fi
;;
-o*) obase=${1#-o}
obase=${obase#=}
if [ -z $obase ] ; then
shift
obase=$1
fi
;;
-s*) sfactors=${1#-s}
sfactors=${sfactors#=}
if [ -z $sfactors ] ; then
shift
sfactors = $1
fi
;;
-t*) tfactors=${1#-t}
tfactors=${tfactors#=}
if [ -z $tfactors ] ; then
shift
sfactors = $1
fi
;;
-x) nottable=1;;
-h) usage; exit 0;;
*) pargs=(${pargs[*]} $1);;
esac
shift
done
if [ -n "$sfactors" ] || [ -n "$tfactors" ] ; then
echo "Factor filtering is not implemented yet!"
exit 2
fi
extract=${pargs[0]}
ibase=${pargs[1]}
L1tag=${pargs[2]}
L2tag=${pargs[3]}
obase=${obase:=$ibase}
fifo=$obase.$$
trap 'cleanup' 0
export LC_ALL=C
if [ -z "$nottable" ] ; then
mkfifo $fifo; sort -S 5G < $fifo | gzip > $obase.fwd.gz &
mkfifo $fifo.inv; sort -S 5G < $fifo.inv | gzip > $obase.bwd.gz &
fi
if [ -n "$dmodel" ] ; then
mkfifo $fifo.o
sort -S 5G < $fifo.o | gzip > $obase.dst.gz &
dspec="orientation --model "
dspec+=`echo $dmodel | perl -pe 's/((hier|phrase|wbe)-(msd|msrl|mono)).*/$1/;'`
fi
txt1=${ibase}.${L1tag}.gz
txt2=${ibase}.${L2tag}.gz
aln=${ibase}.aln.gz
echo "($extract <(zcat -f $txt1) <(zcat -f $txt2) <(zcat -f $aln) $fifo $plen $dspec) || exit 1"
($extract <(zcat -f $txt2) <(zcat -f $txt1) <(zcat -f $aln) $fifo $plen $dspec) || exit 1
wait
|