translation/tools/mosesdecoder/scripts/training/rdlm/extract_syntactic_ngrams.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Rico Sennrich
#
# This file is part of moses.  Its use is licensed under the GNU Lesser General
# Public License version 2.1 or, at your option, any later version.

"""
Extract syntactic n-grams from dependency treebank in Moses XML format for
training RDLM.

Expected format can be produced with
mosesdecoder/scripts/training/wrapper/conll2mosesxml.py

OOV terminal symbols are mapped to preterminal; OOV nonterminals are mapped
to 0 (<unk>)
"""

from __future__ import print_function, unicode_literals, division
import sys
import codecs
import argparse

# Hack for python2/3 compatibility
from io import open

argparse.open = open

try:
    from lxml import etree as ET
except ImportError:
    from xml.etree import cElementTree as ET


def create_parser():
    parser = argparse.ArgumentParser(
        description=(
            "Extract syntactic n-grams from parsed corpus in "
            "Moses XML format for training RDLM"))

    parser.add_argument(
        '--input', '-i', type=argparse.FileType('r'), default=sys.stdin,
        metavar='PATH',
        help='Input file (default: standard input).')
    parser.add_argument(
        '--output', '-o', type=argparse.FileType('w'), default=sys.stdout,
        metavar='PATH',
        help='Output file (default: standard output).')
    parser.add_argument(
        '--mode', type=str, choices=['label', 'head'], required=True,
        help='Predict terminals (head) or dependency labels (label).')
    parser.add_argument(
        '--vocab', metavar='PATH', type=str, required=True,
        help=(
            "Input layer vocabulary file (one item per line; "
            "first line '<unk>')"))
    parser.add_argument(
        '--output_vocab', metavar='PATH', type=str,
        help=(
            "Output layer vocabulary file "
            "(default: use input layer vocabulary)"))
    parser.add_argument(
        '--left_context', metavar='INT', type=int, default=3,
        help=(
            "Size of context vector for left siblings "
            "(default: %(default)s)"))
    parser.add_argument(
        '--right_context', metavar='INT', type=int, default=0,
        help=(
            "Size of context vector for right siblings "
            "(default: %(default)s)"))
    parser.add_argument(
        '--up_context', metavar='INT', type=int, default=2,
        help=(
            "Size of context vector for ancestors "
            "(default: %(default)s)"))
    parser.add_argument(
        '--glue_symbol', metavar='STR', type=str, default='Q',
        help=(
            "Glue symbol. Will be skipped during extraction "
            "(default: %(default)s)"))
    parser.add_argument(
        '--start_symbol', metavar='STR', type=str, default='SSTART',
        help=(
            "Sentence start symbol. Will be skipped during extraction "
            "(default: %(default)s)"))
    parser.add_argument(
        '--end_symbol', metavar='STR', type=str, default='SEND',
        help=(
            "Sentence end symbol. Will be skipped during extraction "
            "(default: %(default)s)"))
    return parser


def escape_text(s):

    s = s.replace('|', '&#124;')  # factor separator
    s = s.replace('[', '&#91;')  # syntax non-terminal
    s = s.replace(']', '&#93;')  # syntax non-terminal
    s = s.replace('\'', '&apos;')  # xml special character
    s = s.replace('"', '&quot;')  # xml special character
    return s


def get_head(xml):
    """Deterministic heuristic to get head of subtree."""
    head = None
    preterminal = None
    for child in xml:
        if not len(child):
            preterminal = child.get('label')
            head = escape_text(child.text.strip())
            return head, preterminal

    return head, preterminal


def get_syntactic_ngrams(xml, options, vocab, output_vocab,
                         parent_heads=None, parent_labels=None):

    if len(xml):

        # Skip glue rules.
        skip_glue_labels = [
            options.glue_symbol,
            options.start_symbol,
            options.end_symbol,
            ]
        if xml.get('label') in skip_glue_labels:
            for child in xml:
                get_syntactic_ngrams(
                    child, options, vocab, output_vocab, parent_heads,
                    parent_labels)
            return

        # Skip virtual nodes.
        skip_virtual_labels = [
            '<stop_label>',
            '<start_label>',
            ]
        if xml.get('label') in skip_virtual_labels:
            return

        if not parent_heads:
            parent_heads = (
                [vocab.get('<root_head>', 0)] * options.up_context)
            parent_labels = (
                [vocab.get('<root_label>', 0)] * options.up_context)

            head, preterminal = get_head(xml)
            if not head:
                head = '<dummy_head>'
                preterminal = head
            elif head not in vocab:
                head = preterminal

            label = xml.get('label')

            # syntactic n-gram for root node
            int_list = []
            int_list.extend([start_head_idx] * options.left_context)
            int_list.extend([start_label_idx] * options.left_context)
            int_list.extend([stop_head_idx] * options.right_context)
            int_list.extend([stop_label_idx] * options.right_context)
            int_list.extend(parent_heads)
            int_list.extend(parent_labels)

            # write root of tree
            if options.mode == 'label':
                int_list.append(output_vocab.get(label, 0))
                options.output.write(' '.join(map(str, int_list)) + '\n')
            elif options.mode == 'head' and not head == '<dummy_head>':
                int_list.append(vocab.get(label, 0))
                int_list.append(
                    output_vocab.get(head, output_vocab.get(preterminal, 0)))
                options.output.write(' '.join(map(str, int_list)) + '\n')

            parent_heads.append(vocab.get(head, 0))
            parent_labels.append(vocab.get(label, 0))

        # virtual start/end-of-subtree tag
        if len(xml) > 0:
            if options.right_context:
                start = ET.Element('tree')
                start2 = ET.Element('tree')
                start.set('label', '<start_label>')
                start2.set('label', 'XY')
                start2.text = '<start_head>'
                start.append(start2)
                xml.insert(0, start)
            if options.left_context:
                end = ET.Element('tree')
                end2 = ET.Element('tree')
                end.set('label', '<stop_label>')
                end2.set('label', 'XY')
                end2.text = '<stop_head>'
                end.append(end2)
                xml.append(end)

        heads = []
        preterminals = []
        labels = []

        for child in xml:
            if not len(child):
                # Mark that the previous sibling is the head of the
                # structure (the head/label are not repeated because they're
                # also head/label of the parent).
                head_child = '<head_head>'
                preterminal_child = head_child
                child_label = '<head_label>'
            else:
                head_child, preterminal_child = get_head(child)
                child_label = child.get('label')

            if head_child is None:
                head_child = '<dummy_head>'

            heads.append(head_child)
            preterminals.append(preterminal_child)
            labels.append(child_label)

            heads_idx = [
                vocab.get(heads[i], vocab.get(preterminals[i], 0))
                for i in range(len(heads))]
            labels_idx = [
                vocab.get(labels[i], 0)
                for i in range(len(labels))]

        # Ancestor context is the same for all children.
        up_heads = parent_heads[-options.up_context:]
        up_labels = parent_labels[-options.up_context:]

        skip_special_heads = [
            '<dummy_head>',
            '<head_head>',
            '<stop_head>',
            '<start_head>',
            ]
        for i, child in enumerate(xml):

            # Skip some special symbols, but recursively extract n-grams
            # for its children.
            if options.mode == 'head' and heads[i] in skip_special_heads:
                parent_heads.append(vocab.get(heads[i], 0))
                parent_labels.append(vocab.get(labels[i], 0))
                get_syntactic_ngrams(
                    child, options, vocab, output_vocab, parent_heads,
                    parent_labels)
                parent_heads.pop()
                parent_labels.pop()
                continue

            previous_heads = heads_idx[max(0, i - options.left_context):i]
            previous_labels = labels_idx[max(0, i - options.left_context):i]

            subsequent_heads = heads_idx[i + 1:i + options.right_context + 1]
            subsequent_labels = labels_idx[i + 1:i + options.right_context + 1]

            if len(previous_heads) < options.left_context:
                previous_heads = (
                    [start_head_idx] *
                    (options.left_context - len(previous_heads)) +
                    previous_heads)
                previous_labels = (
                    [start_label_idx] *
                    (options.left_context - len(previous_labels)) +
                    previous_labels)

            if len(subsequent_heads) < options.right_context:
                subsequent_heads += (
                    [stop_head_idx] *
                    (options.right_context - len(subsequent_heads)))
                subsequent_labels += (
                    [stop_label_idx] *
                    (options.right_context - len(subsequent_labels)))

            int_list = []
            int_list.extend(previous_heads)
            int_list.extend(previous_labels)
            int_list.extend(subsequent_heads)
            int_list.extend(subsequent_labels)
            int_list.extend(up_heads)
            int_list.extend(up_labels)
            if options.mode == 'label':
                int_list.append(output_vocab.get(labels[i], 0))
            elif options.mode == 'head':
                int_list.append(vocab.get(labels[i], 0))
                int_list.append(
                    output_vocab.get(
                        heads[i], output_vocab.get(preterminals[i], 0)))

            options.output.write(' '.join(map(str, int_list)) + '\n')

            parent_heads.append(
                vocab.get(heads[i], vocab.get(preterminals[i], 0)))
            parent_labels.append(vocab.get(labels[i], 0))

            get_syntactic_ngrams(
                child, options, vocab, output_vocab, parent_heads,
                parent_labels)

            parent_heads.pop()
            parent_labels.pop()


def load_vocab(path):
    v = {}
    for i, line in enumerate(open(path, encoding="UTF-8")):
        v[line.strip()] = i
    return v


def main(options):
    vocab = load_vocab(options.vocab)

    if options.output_vocab is None:
        sys.stderr.write(
            "No output vocabulary specified; using input vocabulary.\n")
        output_vocab = vocab
    else:
        output_vocab = load_vocab(options.output_vocab)

    global start_head_idx
    global start_label_idx
    global stop_head_idx
    global stop_label_idx
    start_head_idx = vocab.get("<start_head>", 0)
    start_label_idx = vocab.get("<start_label>", 0)
    stop_head_idx = vocab.get("<stop_head>", 0)
    stop_label_idx = vocab.get("<stop_label>", 0)

    i = 0
    for line in options.input:
        if i and not i % 50000:
            sys.stderr.write('.')
        if i and not i % 1000000:
            sys.stderr.write('{0}\n'.format(i))
        if sys.version_info < (3, 0):
            if line == b'\n':
                continue
            # hack for older moses versions with inconsistent encoding of "|"
            line = line.replace(b'&bar;', b'&#124;')
        else:
            if line == '\n':
                continue
            # hack for older moses versions with inconsistent encoding of "|"
            line = line.replace('&bar;', '&#124;')
        xml = ET.fromstring(line)
        get_syntactic_ngrams(xml, options, vocab, output_vocab)
        i += 1

if __name__ == '__main__':

    if sys.version_info < (3, 0):
        sys.stderr = codecs.getwriter('UTF-8')(sys.stderr)
        sys.stdout = codecs.getwriter('UTF-8')(sys.stdout)

    parser = create_parser()
    options = parser.parse_args()

    main(options)