translation/tools/mosesdecoder/scripts/server/moses.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of moses.  Its use is licensed under the GNU Lesser General
# Public License version 2.1 or, at your option, any later version.

"""
Python utilities for moses

This package mostly wraps standard Moses utilities into pipes.

Written by Ulrich Germann

This package borrows from scripts written by Christian Buck

The package assumes that there is a complete moses installation
(including scripts) under one root directory,
e.g., via ::
    bjam --with-xmlrpc-c=... [...] --install-scripts --prefix=${HOME}/moses
By default, this root directory is "${HOME}/moses".
"""

import os
import sys
import time
import xmlrpclib
from subprocess import (
    PIPE,
    Popen,
    )


moses_root = os.environ.get('MOSES_ROOT', os.environ.get('HOME') + "/moses")


class ProcessWrapper:

    def __init__(self, cmd=[]):
        self.process = None
        self.cmd = cmd

    def start(self, stdin=PIPE, stdout=PIPE):
        if self.process:
            raise Exception("Process is already running")
        self.process = Popen(self.cmd, stdin=stdin, stdout=stdout)

    def __del__(self):
        if self.process:
            self.process.terminate()


class LineProcessor(ProcessWrapper):

    def __call__(self, input):
        if not self.process:
            self.start()
        self.process.stdin.write("%s\n" % input.strip())
        self.process.stdin.flush()
        return self.process.stdout.readline().strip()


class SentenceSplitter(ProcessWrapper):
    """Wrapper for standard Moses sentence splitter."""

    def __init__(self, lang):
        ssplit_cmd = moses_root + "/scripts/ems/support/split-sentences.perl"
        self.cmd = [ssplit_cmd, "-b", "-q", "-l", lang]
        self.process = None

    def __call__(self, input):
        if not self.process:
            self.start()
        self.process.stdin.write(input.strip() + "\n<P>\n")
        self.process.stdin.flush()
        x = self.process.stdout.readline().strip()
        ret = []
        while x != '<P>' and x != '':
            ret.append(x)
            x = self.process.stdout.readline().strip()
        return ret


class Pretokenizer(LineProcessor):
    """Pretokenizer wrapper.

    The pretokenizer fixes known issues with the input.
    """
    def __init__(self, lang):
        pretok_cmd = moses_root + "/scripts/tokenizer/pre-tokenizer.perl"
        self.cmd = [pretok_cmd, "-b", "-q", "-l", lang]
        self.process = None


class Tokenizer(LineProcessor):
    """Tokenizer wrapper.

    The pretokenizer fixes known issues with the input.
    """
    def __init__(self, lang, args=["-a", "-no-escape"]):
        tok_cmd = moses_root + "/scripts/tokenizer/tokenizer.perl"
        self.cmd = [tok_cmd, "-b", "-q", "-l", lang] + args
        self.process = None


class Truecaser(LineProcessor):
    """Truecaser wrapper."""
    def __init__(self, model):
        truecase_cmd = moses_root + "/scripts/recaser/truecase.perl"
        self.cmd = [truecase_cmd, "-b", "--model", model]
        self.process = None


class LineProcessorPipeline:
    """Line processor: one line in, one line out."""
    def __init__(self, parts=[]):
        self.chain = [LineProcessor(p.cmd) for p in parts]

    def start(self):
        if len(self.chain) == 0:
            return
        if self.chain[0].process:
            return
        self.chain[0].start()
        for i in xrange(1, len(self.chain)):
            self.chain[i].start(stdin=self.chain[i - 1].process.stdout)

    def __call__(self, input):
        if len(self.chain) == 0:
            return input
        self.start()
        self.chain[0].process.stdin.write("%s\n" % input.strip())
        self.chain[0].process.stdin.flush()
        return self.chain[0].process.stdout.readline().strip()


def find_free_port(p):
    """Find a free port, starting at /p/.

    :return: The free port, or False if none found.
    """
    ret = p
    while ret - p < 20:
        devnull = open(os.devnull, "w")
        n = Popen(["netstat", "-tnp"], stdout=PIPE, stderr=devnull)
        if n.communicate()[0].find(":%d " % ret) < 0:
            return p
        ret += 1
    return False


class MosesServer(ProcessWrapper):

    def __init__(self, args=[]):
        self.process = None
        mserver_cmd = moses_root + "/bin/mosesserver"
        self.cmd = [mserver_cmd] + args
        self.url = None
        self.proxy = None

    def start(self, config=None, args=[], port=7447, debug=False):
        self.cmd.extend(args)
        if config:
            if "-f" in args:
                raise Exception("Config file specified twice")
            else:
                self.cmd.extend(["-f", config])
        self.port = port  # find_free_port(port)
        if not self.port:
            raise Exception("Cannot find free port for moses server!")
        self.cmd.extend(["--server-port", "%d" % self.port])
        if debug:
            print >>sys.stderr, self.cmd
            # self.stderr = open("mserver.%d.stderr"%self.port,'w')
            # self.stdout = open("mserver.%d.stdout"%self.port,'w')
            # self.process = Popen(
            #     self.cmd, stderr=self.stderr, stdout=self.stdout)
            self.process = Popen(self.cmd)
        else:
            devnull = open(os.devnull, "w")
            self.process = Popen(self.cmd, stderr=devnull, stdout=devnull)

        if self.process.poll():
            raise Exception("FATAL ERROR: Could not launch moses server!")
        if debug:
            print >>sys.stderr, "MOSES port is %d." % self.port
            print >>sys.stderr, "Moses poll status is", self.process.poll()

        self.url = "http://localhost:%d/RPC2" % self.port
        self.connect(self.url)

        return True

    def connect(self, url):
        if url[:4] != "http":
            url = "http://%s" % url
        if url[-5:] != "/RPC2":
            url += "/RPC2"
        self.url = url
        self.proxy = xmlrpclib.ServerProxy(self.url)

    def translate(self, input):
        attempts = 0
        while attempts < 100:
            try:
                if type(input) is unicode:
                    # If the server does not expect unicode, provide a
                    # properly encoded string!
                    param = {'text': input.strip().encode('utf8')}
                    return self.proxy.translate(param)['text'].decode('utf8')

                elif type(input) is str:
                    param = {'text': input.strip()}
                    return self.proxy.translate(param)['text']

                elif type(input) is list:
                    return [self.translate(x) for x in input]

                elif type(input) is dict:
                    return self.proxy.translate(input)

                else:
                    raise Exception("Can't handle input of this type!")

            except:
                attempts += 1
                print >>sys.stderr, "WAITING", attempts
                time.sleep(1)
        raise Exception("Translation request failed")