translation/tools/mosesdecoder/phrase-extract/extract-mixed-syntax/AlignedSentence.cpp

/*
 * AlignedSentence.cpp
 *
 *  Created on: 18 Feb 2014
 *      Author: s0565741
 */

#include <sstream>
#include "moses/Util.h"
#include "AlignedSentence.h"
#include "Parameter.h"

using namespace std;


/////////////////////////////////////////////////////////////////////////////////
AlignedSentence::AlignedSentence(int lineNum,
                                 const std::string &source,
                                 const std::string &target,
                                 const std::string &alignment)
  :m_lineNum(lineNum)
{
  PopulateWordVec(m_source, source);
  PopulateWordVec(m_target, target);
  PopulateAlignment(alignment);
}

AlignedSentence::~AlignedSentence()
{
  Moses::RemoveAllInColl(m_source);
  Moses::RemoveAllInColl(m_target);
}

void AlignedSentence::PopulateWordVec(Phrase &vec, const std::string &line)
{
  std::vector<string> toks;
  Moses::Tokenize(toks, line);

  vec.resize(toks.size());
  for (size_t i = 0; i < vec.size(); ++i) {
    const string &tok = toks[i];
    Word *word = new Word(i, tok);
    vec[i] = word;
  }
}

void AlignedSentence::PopulateAlignment(const std::string &line)
{
  vector<string> alignStr;
  Moses::Tokenize(alignStr, line);

  for (size_t i = 0; i < alignStr.size(); ++i) {
    vector<int> alignPair;
    Moses::Tokenize(alignPair, alignStr[i], "-");
    assert(alignPair.size() == 2);

    int sourcePos = alignPair[0];
    int targetPos = alignPair[1];

    if (sourcePos >= m_source.size()) {
      cerr << "ERROR1:AlignedSentence=" << Debug() << endl;
      cerr << "m_source=" << m_source.size() << endl;
      abort();
    }
    assert(sourcePos < m_source.size());
    assert(targetPos < m_target.size());
    Word *sourceWord = m_source[sourcePos];
    Word *targetWord = m_target[targetPos];

    sourceWord->AddAlignment(targetWord);
    targetWord->AddAlignment(sourceWord);
  }
}

std::string AlignedSentence::Debug() const
{
  stringstream out;
  out << "m_lineNum:";
  out << m_lineNum;
  out << endl;

  out << "m_source:";
  out << m_source.Debug();
  out << endl;

  out << "m_target:";
  out << m_target.Debug();
  out << endl;

  out << "consistent phrases:" << endl;
  out << m_consistentPhrases.Debug();
  out << endl;

  return out.str();
}

std::vector<int> AlignedSentence::GetSourceAlignmentCount() const
{
  vector<int> ret(m_source.size());

  for (size_t i = 0; i < m_source.size(); ++i) {
    const Word &word = *m_source[i];
    ret[i] = word.GetAlignmentIndex().size();
  }
  return ret;
}

void AlignedSentence::Create(const Parameter &params)
{
  CreateConsistentPhrases(params);
  m_consistentPhrases.AddHieroNonTerms(params);
}

void AlignedSentence::CreateConsistentPhrases(const Parameter &params)
{
  int countT = m_target.size();
  int countS = m_source.size();

  m_consistentPhrases.Initialize(countS);

  // check alignments for target phrase startT...endT
  for(int lengthT=1;
      lengthT <= params.maxSpan && lengthT <= countT;
      lengthT++) {
    for(int startT=0; startT < countT-(lengthT-1); startT++) {

      // that's nice to have
      int endT = startT + lengthT - 1;

      // find find aligned source words
      // first: find minimum and maximum source word
      int minS = 9999;
      int maxS = -1;
      vector< int > usedS = GetSourceAlignmentCount();
      for(int ti=startT; ti<=endT; ti++) {
        const Word &word = *m_target[ti];
        const std::set<int> &alignment = word.GetAlignmentIndex();

        std::set<int>::const_iterator iterAlign;
        for(iterAlign = alignment.begin(); iterAlign != alignment.end(); ++iterAlign) {
          int si = *iterAlign;
          if (si<minS) {
            minS = si;
          }
          if (si>maxS) {
            maxS = si;
          }
          usedS[ si ]--;
        }
      }

      // unaligned phrases are not allowed
      if( maxS == -1 )
        continue;

      // source phrase has to be within limits
      size_t width = maxS - minS + 1;

      if( width < params.minSpan )
        continue;

      if( width > params.maxSpan )
        continue;

      // check if source words are aligned to out of bound target words
      bool out_of_bounds = false;
      for(int si=minS; si<=maxS && !out_of_bounds; si++)
        if (usedS[si]>0) {
          out_of_bounds = true;
        }

      // if out of bound, you gotta go
      if (out_of_bounds)
        continue;

      // done with all the checks, lets go over all consistent phrase pairs
      // start point of source phrase may retreat over unaligned
      for(int startS=minS;
          (startS>=0 &&
           startS>maxS - params.maxSpan && // within length limit
           (startS==minS || m_source[startS]->GetAlignment().size()==0)); // unaligned
          startS--) {
        // end point of source phrase may advance over unaligned
        for(int endS=maxS;
            (endS<countS && endS<startS + params.maxSpan && // within length limit
             (endS==maxS || m_source[endS]->GetAlignment().size()==0)); // unaligned
            endS++) {

          // take note that this is a valid phrase alignment
          m_consistentPhrases.Add(startS, endS, startT, endT, params);
        }
      }
    }
  }
}