translation/tools/mosesdecoder/phrase-extract/extract-mixed-syntax/Rules.cpp

/*
 * Rules.cpp
 *
 *  Created on: 20 Feb 2014
 *      Author: hieu
 */

#include <sstream>
#include "Rules.h"
#include "ConsistentPhrase.h"
#include "ConsistentPhrases.h"
#include "AlignedSentence.h"
#include "Rule.h"
#include "Parameter.h"
#include "moses/Util.h"

using namespace std;

extern bool g_debug;

Rules::Rules(const AlignedSentence &alignedSentence)
  :m_alignedSentence(alignedSentence)
{
}

Rules::~Rules()
{
  Moses::RemoveAllInColl(m_keepRules);
}

void Rules::CreateRules(const ConsistentPhrase &cp,
                        const Parameter &params)
{
  if (params.hieroSourceLHS) {
    const NonTerm &nonTerm = cp.GetHieroNonTerm();
    CreateRule(nonTerm, params);
  } else {
    const ConsistentPhrase::NonTerms &nonTerms = cp.GetNonTerms();
    for (size_t i = 0; i < nonTerms.size(); ++i) {
      const NonTerm &nonTerm = nonTerms[i];
      CreateRule(nonTerm, params);
    }
  }
}

void Rules::CreateRule(const NonTerm &nonTerm,
                       const Parameter &params)
{
  Rule *rule = new Rule(nonTerm, m_alignedSentence);

  rule->Prevalidate(params);
  rule->CreateTarget(params);
  rule->CreateProperties(params);

  if (rule->CanRecurse()) {
    Extend(*rule, params);
  }

  if (rule->IsValid()) {
    m_keepRules.insert(rule);
  } else {
    delete rule;
  }

}

void Rules::Extend(const Parameter &params)
{
  const ConsistentPhrases &allCPS = m_alignedSentence.GetConsistentPhrases();

  size_t size = m_alignedSentence.GetPhrase(Moses::Input).size();
  for (size_t sourceStart = 0; sourceStart < size; ++sourceStart) {
    for (size_t sourceEnd = sourceStart; sourceEnd < size; ++sourceEnd) {
      const ConsistentPhrases::Coll &cps = allCPS.GetColl(sourceStart, sourceEnd);

      ConsistentPhrases::Coll::const_iterator iter;
      for (iter = cps.begin(); iter != cps.end(); ++iter) {
        const ConsistentPhrase &cp = **iter;
        CreateRules(cp, params);
      }
    }
  }
}

void Rules::Extend(const Rule &rule, const Parameter &params)
{
  const ConsistentPhrases &allCPS = m_alignedSentence.GetConsistentPhrases();
  int sourceMin = rule.GetNextSourcePosForNonTerm();

  int ruleStart = rule.GetConsistentPhrase().corners[0];
  int ruleEnd = rule.GetConsistentPhrase().corners[1];

  for (int sourceStart = sourceMin; sourceStart <= ruleEnd; ++sourceStart) {
    for (int sourceEnd = sourceStart; sourceEnd <= ruleEnd; ++sourceEnd) {
      if (sourceStart == ruleStart && sourceEnd == ruleEnd) {
        // don't cover whole rule with 1 non-term
        continue;
      }

      const ConsistentPhrases::Coll &cps = allCPS.GetColl(sourceStart, sourceEnd);
      Extend(rule, cps, params);
    }
  }
}

void Rules::Extend(const Rule &rule, const ConsistentPhrases::Coll &cps, const Parameter &params)
{
  ConsistentPhrases::Coll::const_iterator iter;
  for (iter = cps.begin(); iter != cps.end(); ++iter) {
    const ConsistentPhrase &cp = **iter;
    Extend(rule, cp, params);
  }
}

void Rules::Extend(const Rule &rule, const ConsistentPhrase &cp, const Parameter &params)
{
  const ConsistentPhrase::NonTerms &nonTerms = cp.GetNonTerms();
  for (size_t i = 0; i < nonTerms.size(); ++i) {
    const NonTerm &nonTerm = nonTerms[i];

    Rule *newRule = new Rule(rule, nonTerm);
    newRule->Prevalidate(params);
    newRule->CreateTarget(params);
    newRule->CreateProperties(params);

    if (newRule->CanRecurse()) {
      // recursively extend
      Extend(*newRule, params);
    }

    if (newRule->IsValid()) {
      m_keepRules.insert(newRule);
    } else {
      delete newRule;
    }
  }
}

std::string Rules::Debug() const
{
  stringstream out;

  std::set<Rule*>::const_iterator iter;
  out << "m_keepRules:" << endl;
  for (iter = m_keepRules.begin(); iter != m_keepRules.end(); ++iter) {
    const Rule &rule = **iter;
    out << rule.Debug() << endl;
  }

  return out.str();
}

void Rules::Output(std::ostream &out, bool forward, const Parameter &params) const
{
  std::set<Rule*, CompareRules>::const_iterator iter;
  for (iter = m_mergeRules.begin(); iter != m_mergeRules.end(); ++iter) {
    const Rule &rule = **iter;
    rule.Output(out, forward);
    out << endl;
  }
}

void Rules::Consolidate(const Parameter &params)
{
  if (params.fractionalCounting) {
    CalcFractionalCount();
  } else {
    std::set<Rule*>::iterator iter;
    for (iter = m_keepRules.begin(); iter != m_keepRules.end(); ++iter) {
      Rule &rule = **iter;
      rule.SetCount(1);
    }
  }

  MergeRules(params);
}

void Rules::MergeRules(const Parameter &params)
{
  typedef std::set<Rule*, CompareRules> MergeRules;

  std::set<Rule*>::const_iterator iterOrig;
  for (iterOrig = m_keepRules.begin(); iterOrig != m_keepRules.end(); ++iterOrig) {
    Rule *origRule = *iterOrig;

    pair<MergeRules::iterator, bool> inserted = m_mergeRules.insert(origRule);
    if (!inserted.second) {
      // already there, just add count
      Rule &rule = **inserted.first;
      float newCount = rule.GetCount() + origRule->GetCount();
      rule.SetCount(newCount);
    }
  }
}

void Rules::CalcFractionalCount()
{
  typedef std::set<Rule*> RuleColl;
  typedef std::map<const ConsistentPhrase*, RuleColl> RuleByConsistentPhrase;
  RuleByConsistentPhrase allRules;

  // sort by source AND target ranges
  std::set<Rule*>::const_iterator iter;
  for (iter = m_keepRules.begin(); iter != m_keepRules.end(); ++iter) {
    Rule *rule = *iter;
    const ConsistentPhrase &cp = rule->GetConsistentPhrase();
    RuleColl &ruleColl =  allRules[&cp];
    ruleColl.insert(rule);
  }

  // fractional count
  RuleByConsistentPhrase::iterator iterOuter;
  for (iterOuter = allRules.begin(); iterOuter != allRules.end(); ++iterOuter) {
    RuleColl &rules = iterOuter->second;

    RuleColl::iterator iterInner;
    for (iterInner = rules.begin(); iterInner != rules.end(); ++iterInner) {
      Rule &rule = **iterInner;
      rule.SetCount(1.0f / (float) rules.size());
    }
  }

}